diff --git a/Meadowlark-js/.env.example b/Meadowlark-js/.env.example index 70ee16b8..508530d5 100644 --- a/Meadowlark-js/.env.example +++ b/Meadowlark-js/.env.example @@ -7,6 +7,8 @@ OAUTH_SIGNING_KEY="" MONGODB_MAX_NUMBER_OF_RETRIES=1 +# Set the maximum number of retries for PostgreSql. +POSTGRES_MAX_NUMBER_OF_RETRIES=1 # # The settings below are typically good enough to get started # diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json index 11548530..c450e82c 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json @@ -22,6 +22,7 @@ "@edfi/meadowlark-authz-server": "^v0.3.6-pre-36", "@edfi/meadowlark-core": "^v0.3.6-pre-36", "@edfi/meadowlark-utilities": "^v0.3.6-pre-36", + "async-retry": "^1.3.3", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts index af4fecc1..b8f67b30 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts @@ -3,7 +3,8 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. -import { Logger } from '@edfi/meadowlark-utilities'; +import retry from 'async-retry'; +import { Logger, Config } from '@edfi/meadowlark-utilities'; import type { DeleteResult, DeleteRequest, ReferringDocumentInfo, MeadowlarkId } from '@edfi/meadowlark-core'; import type { PoolClient } from 'pg'; import { @@ -26,91 +27,128 @@ export async function deleteDocumentByDocumentUuid( client: PoolClient, ): Promise { Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid ${documentUuid}`, traceId); - - let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' }; - let meadowlarkId: MeadowlarkId = '' as MeadowlarkId; + let retryCount = 0; try { - await beginTransaction(client); - // Find the alias meadowlarkIds for the document to be deleted - const documentAliasIdsResult: MeadowlarkDocumentIdAndAliasId[] = await findAliasMeadowlarkIdsForDocumentByDocumentUuid( - client, - documentUuid, - ); - // Each row contains documentUuid and corresponding meadowlarkId (meadowlark_id), - // we just need the first row to return the meadowlark_id - meadowlarkId = documentAliasIdsResult.length > 0 ? documentAliasIdsResult[0].meadowlark_id : ('' as MeadowlarkId); - if (validateNoReferencesToDocument) { - // All documents have alias meadowlarkIds. If no alias meadowlarkIds were found, the document doesn't exist - if (meadowlarkId === '') { - await rollbackTransaction(client); - Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid: DocumentUuid ${documentUuid} does not exist`, traceId); - deleteResult = { response: 'DELETE_FAILURE_NOT_EXISTS' }; - return deleteResult; - } - - // Extract from the query result - const documentAliasMeadowlarkIds: MeadowlarkId[] = documentAliasIdsResult.map((ref) => ref.alias_meadowlark_id); - - // Find any documents that reference this document, either it's own meadowlarkId or an alias - const referenceResult: MeadowlarkId[] = await findReferencingMeadowlarkIds(client, documentAliasMeadowlarkIds); - - const references = referenceResult.filter((ref) => ref !== meadowlarkId); - - // If this document is referenced, it's a validation failure - if (references.length > 0) { - Logger.debug( - `${moduleName}.deleteDocumentByDocumentUuid: Deleting document meadowlarkId ${meadowlarkId} failed due to existing references`, - traceId, - ); - - // Get the information of up to five referring documents for failure message purposes - const referenceIds = references.map((ref) => ref); - const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting( - client, - referenceIds, - ); - - if (referringDocumentInfo.length === 0) { + const numberOfRetries: number = Config.get('POSTGRES_MAX_NUMBER_OF_RETRIES'); + const deleteProcess = await retry( + async (bail) => { + let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' }; + let meadowlarkId: MeadowlarkId = '' as MeadowlarkId; + try { + await beginTransaction(client); + // Find the alias meadowlarkIds for the document to be deleted + const documentAliasIdsResult: MeadowlarkDocumentIdAndAliasId[] = + await findAliasMeadowlarkIdsForDocumentByDocumentUuid(client, documentUuid); + // Each row contains documentUuid and corresponding meadowlarkId (meadowlark_id), + // we just need the first row to return the meadowlark_id + meadowlarkId = documentAliasIdsResult.length > 0 ? documentAliasIdsResult[0].meadowlark_id : ('' as MeadowlarkId); + if (validateNoReferencesToDocument) { + // All documents have alias meadowlarkIds. If no alias meadowlarkIds were found, the document doesn't exist + if (meadowlarkId === '') { + await rollbackTransaction(client); + Logger.debug( + `${moduleName}.deleteDocumentByDocumentUuid: DocumentUuid ${documentUuid} does not exist`, + traceId, + ); + deleteResult = { response: 'DELETE_FAILURE_NOT_EXISTS' }; + return deleteResult; + } + + // Extract from the query result + const documentAliasMeadowlarkIds: MeadowlarkId[] = documentAliasIdsResult.map((ref) => ref.alias_meadowlark_id); + + // Find any documents that reference this document, either it's own meadowlarkId or an alias + const referenceResult: MeadowlarkId[] = await findReferencingMeadowlarkIds(client, documentAliasMeadowlarkIds); + + const references = referenceResult.filter((ref) => ref !== meadowlarkId); + + // If this document is referenced, it's a validation failure + if (references.length > 0) { + Logger.debug( + `${moduleName}.deleteDocumentByDocumentUuid: Deleting document meadowlarkId ${meadowlarkId} failed due to existing references`, + traceId, + ); + + // Get the information of up to five referring documents for failure message purposes + const referenceIds = references.map((ref) => ref); + const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting( + client, + referenceIds, + ); + + if (referringDocumentInfo.length === 0) { + await rollbackTransaction(client); + const errorMessage = `${moduleName}.deleteDocumentByDocumentUuid Error retrieving documents referenced by ${meadowlarkId}, a null result set was returned`; + deleteResult.failureMessage = errorMessage; + Logger.error(errorMessage, traceId); + return deleteResult; + } + + deleteResult = { + response: 'DELETE_FAILURE_REFERENCE', + referringDocumentInfo, + }; + await rollbackTransaction(client); + return deleteResult; + } + } + + // Perform the document delete + Logger.debug( + `${moduleName}.deleteDocumentByDocumentUuid: Deleting document documentUuid ${documentUuid}`, + traceId, + ); + deleteResult = await deleteDocumentRowByDocumentUuid(client, documentUuid); + + // Delete references where this is the parent document + Logger.debug( + `${moduleName}.deleteDocumentByDocumentUuid Deleting references with documentUuid ${documentUuid} as parent meadowlarkId`, + traceId, + ); + await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId); + + // Delete this document from the aliases table + Logger.debug( + `${moduleName}.deleteDocumentByDocumentUuid Deleting alias entries with meadowlarkId ${meadowlarkId}`, + traceId, + ); + await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId); + + await commitTransaction(client); + } catch (error) { + retryCount += 1; await rollbackTransaction(client); - const errorMessage = `${moduleName}.deleteDocumentByDocumentUuid Error retrieving documents referenced by ${meadowlarkId}, a null result set was returned`; - deleteResult.failureMessage = errorMessage; - Logger.error(errorMessage, traceId); - return deleteResult; + // If there's a serialization failure, rollback the transaction + if (error.code === '40001') { + if (retryCount >= numberOfRetries) { + throw Error('Error after maximum retries'); + } + // Throws the error to be handled by async-retry + throw error; + } else { + // If it's not a serialization failure, don't retry and rethrow the error + Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, error); + // Throws the error to be handled by the caller + bail(error); + } } - - deleteResult = { - response: 'DELETE_FAILURE_REFERENCE', - referringDocumentInfo, - }; - await rollbackTransaction(client); return deleteResult; - } - } - - // Perform the document delete - Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid: Deleting document documentUuid ${documentUuid}`, traceId); - deleteResult = await deleteDocumentRowByDocumentUuid(client, documentUuid); - - // Delete references where this is the parent document - Logger.debug( - `${moduleName}.deleteDocumentByDocumentUuid Deleting references with documentUuid ${documentUuid} as parent meadowlarkId`, - traceId, + }, + { + retries: numberOfRetries, + onRetry: (error, attempt) => { + if (attempt === numberOfRetries) { + Logger.error('Error after maximum retries', error); + } else { + Logger.error('Retrying transaction due to error:', error); + } + }, + }, ); - await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId); - - // Delete this document from the aliases table - Logger.debug( - `${moduleName}.deleteDocumentByDocumentUuid Deleting alias entries with meadowlarkId ${meadowlarkId}`, - traceId, - ); - await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId); - - await commitTransaction(client); - } catch (e) { - Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, e); - await rollbackTransaction(client); - return { response: 'UNKNOWN_FAILURE', failureMessage: '' }; + return deleteProcess; + } catch (error) { + Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, error); + return { response: 'UNKNOWN_FAILURE', failureMessage: error.message ?? '' }; } - return deleteResult; } diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts index 988e275f..4387177a 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts @@ -27,6 +27,7 @@ types.setTypeParser(types.builtins.INT8, (bigintAsString) => parseInt(bigintAsSt */ export async function beginTransaction(client: PoolClient) { await client.query('BEGIN'); + await client.query('SET TRANSACTION ISOLATION LEVEL SERIALIZABLE'); } /** @@ -91,10 +92,7 @@ export async function findAliasMeadowlarkIdsForDocumentByMeadowlarkId( client: PoolClient, meadowlarkId: MeadowlarkId, ): Promise { - const querySelect = format( - `SELECT alias_meadowlark_id FROM meadowlark.aliases WHERE meadowlark_id = %L FOR SHARE NOWAIT`, - meadowlarkId, - ); + const querySelect = format(`SELECT alias_meadowlark_id FROM meadowlark.aliases WHERE meadowlark_id = %L`, meadowlarkId); const queryResult: QueryResult = await client.query(querySelect); return (hasResults(queryResult) ? queryResult.rows.map((ref) => ref.alias_meadowlark_id) : []) as MeadowlarkId[]; } @@ -115,7 +113,7 @@ export async function findAliasMeadowlarkIdsForDocumentByDocumentUuid( documentUuid: DocumentUuid, ): Promise { const querySelect = format( - `SELECT alias_meadowlark_id, meadowlark_id FROM meadowlark.aliases WHERE document_uuid = %L FOR SHARE NOWAIT`, + `SELECT alias_meadowlark_id, meadowlark_id FROM meadowlark.aliases WHERE document_uuid = %L`, documentUuid, ); const queryResult: QueryResult = await client.query(querySelect); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Update.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Update.ts index d5f9cb56..30ea3cc0 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Update.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Update.ts @@ -3,6 +3,7 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +import retry from 'async-retry'; import { MeadowlarkId, UpdateResult, @@ -12,7 +13,7 @@ import { getMeadowlarkIdForSuperclassInfo, ReferringDocumentInfo, } from '@edfi/meadowlark-core'; -import { Logger } from '@edfi/meadowlark-utilities'; +import { Logger, Config } from '@edfi/meadowlark-utilities'; import type { PoolClient } from 'pg'; import { updateDocument, @@ -34,112 +35,147 @@ const moduleName = 'postgresql.repository.Update'; export async function updateDocumentByDocumentUuid(updateRequest: UpdateRequest, client: PoolClient): Promise { const { meadowlarkId, documentUuid, resourceInfo, documentInfo, validateDocumentReferencesExist, traceId } = updateRequest; Logger.info(`${moduleName}.updateDocumentByDocumentUuid ${documentUuid}`, traceId); - let updateResult: UpdateResult = { response: 'UNKNOWN_FAILURE' }; - - const outboundRefs: MeadowlarkId[] = documentInfo.documentReferences.map((dr: DocumentReference) => - getMeadowlarkIdForDocumentReference(dr), - ); + let retryCount = 0; try { - await beginTransaction(client); - - // Get the document to check for staleness and identity change - const documentFromDb: MeadowlarkDocument = await findDocumentByDocumentUuid(client, documentUuid); - - if (documentFromDb === NoMeadowlarkDocument) return { response: 'UPDATE_FAILURE_NOT_EXISTS' }; - - // If request is stale, return conflict - if (documentFromDb.last_modified_at >= documentInfo.requestTimestamp) { - return { response: 'UPDATE_FAILURE_WRITE_CONFLICT' }; - } - - const existingMeadowlarkId: MeadowlarkId = documentFromDb.meadowlark_id; - if (!resourceInfo.allowIdentityUpdates && existingMeadowlarkId !== meadowlarkId) { - updateResult = { response: 'UPDATE_FAILURE_IMMUTABLE_IDENTITY' }; - return updateResult; - } - - if (validateDocumentReferencesExist) { - const failures = await validateReferences( - documentInfo.documentReferences, - documentInfo.descriptorReferences, - outboundRefs, - client, - traceId, - ); - // Abort on validation failure - if (failures.length > 0) { - Logger.debug( - `${moduleName}.updateDocument: Inserting document documentUuid ${documentUuid} failed due to invalid references`, - traceId, - ); - - const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting(client, [ - existingMeadowlarkId, - ]); - - updateResult = { - response: 'UPDATE_FAILURE_REFERENCE', - failureMessage: { error: { message: 'Reference validation failed', failures } }, - referringDocumentInfo, - }; - await rollbackTransaction(client); + const numberOfRetries: number = Config.get('POSTGRES_MAX_NUMBER_OF_RETRIES'); + const updateProcess = await retry( + async (bail) => { + let updateResult: UpdateResult = { response: 'UNKNOWN_FAILURE' }; + try { + const outboundRefs: MeadowlarkId[] = documentInfo.documentReferences.map((dr: DocumentReference) => + getMeadowlarkIdForDocumentReference(dr), + ); + await beginTransaction(client); + + // Get the document to check for staleness and identity change + const documentFromDb: MeadowlarkDocument = await findDocumentByDocumentUuid(client, documentUuid); + + if (documentFromDb === NoMeadowlarkDocument) return { response: 'UPDATE_FAILURE_NOT_EXISTS' }; + + // If request is stale, return conflict + if (documentFromDb.last_modified_at >= documentInfo.requestTimestamp) { + return { response: 'UPDATE_FAILURE_WRITE_CONFLICT' }; + } + + const existingMeadowlarkId: MeadowlarkId = documentFromDb.meadowlark_id; + if (!resourceInfo.allowIdentityUpdates && existingMeadowlarkId !== meadowlarkId) { + updateResult = { response: 'UPDATE_FAILURE_IMMUTABLE_IDENTITY' }; + return updateResult; + } + + if (validateDocumentReferencesExist) { + const failures = await validateReferences( + documentInfo.documentReferences, + documentInfo.descriptorReferences, + outboundRefs, + client, + traceId, + ); + // Abort on validation failure + if (failures.length > 0) { + Logger.debug( + `${moduleName}.updateDocument: Inserting document documentUuid ${documentUuid} failed due to invalid references`, + traceId, + ); + + const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting( + client, + [existingMeadowlarkId], + ); + + updateResult = { + response: 'UPDATE_FAILURE_REFERENCE', + failureMessage: { error: { message: 'Reference validation failed', failures } }, + referringDocumentInfo, + }; + await rollbackTransaction(client); + return updateResult; + } + } + + // Perform the document update + const updateDocumentResult: boolean = await updateDocument(client, updateRequest); + + // Delete existing values from the aliases table + await deleteAliasesForDocumentByMeadowlarkId(client, existingMeadowlarkId); + + // Perform insert of alias meadowlarkIds + await insertAlias(client, documentUuid, meadowlarkId, meadowlarkId); + if (documentInfo.superclassInfo != null) { + const superclassAliasMeadowlarkId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo( + documentInfo.superclassInfo, + ) as MeadowlarkId; + await insertAlias(client, documentUuid, meadowlarkId, superclassAliasMeadowlarkId); + } + + // Delete existing references in references table (by old meadowlarkId) + Logger.debug( + `${moduleName}.upsertDocument: Deleting references for document meadowlarkId ${existingMeadowlarkId}`, + traceId, + ); + await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, existingMeadowlarkId); + + // Adding descriptors to outboundRefs for reference checking + const descriptorOutboundRefs = documentInfo.descriptorReferences.map((dr: DocumentReference) => + getMeadowlarkIdForDocumentReference(dr), + ); + outboundRefs.push(...descriptorOutboundRefs); + + // Perform insert of references to the references table + // eslint-disable-next-line no-restricted-syntax + for (const ref of outboundRefs) { + Logger.debug( + `${moduleName}.upsertDocument: Inserting reference meadowlarkId ${ref} for document meadowlarkId ${meadowlarkId}`, + ref, + ); + await insertOutboundReferences(client, meadowlarkId, ref as MeadowlarkId); + } + + await commitTransaction(client); + + updateResult = updateDocumentResult + ? { + response: 'UPDATE_SUCCESS', + } + : { + response: 'UPDATE_FAILURE_NOT_EXISTS', + }; + return updateResult; + } catch (error) { + retryCount += 1; + await rollbackTransaction(client); + // If there's a serialization failure, rollback the transaction + if (error.code === '40001') { + if (retryCount >= numberOfRetries) { + throw Error('Error after maximum retries'); + } + // Throws the error to be handled by async-retry + throw error; + } else { + // If it's not a serialization failure, don't retry and rethrow the error + Logger.error(`${moduleName}.updateDocumentByDocumentUuid`, traceId, error); + // Throws the error to be handled by the caller + bail(error); + } + } return updateResult; - } - } - - // Perform the document update - const updateDocumentResult: boolean = await updateDocument(client, updateRequest); - - // Delete existing values from the aliases table - await deleteAliasesForDocumentByMeadowlarkId(client, existingMeadowlarkId); - - // Perform insert of alias meadowlarkIds - await insertAlias(client, documentUuid, meadowlarkId, meadowlarkId); - if (documentInfo.superclassInfo != null) { - const superclassAliasMeadowlarkId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo( - documentInfo.superclassInfo, - ) as MeadowlarkId; - await insertAlias(client, documentUuid, meadowlarkId, superclassAliasMeadowlarkId); - } - - // Delete existing references in references table (by old meadowlarkId) - Logger.debug( - `${moduleName}.upsertDocument: Deleting references for document meadowlarkId ${existingMeadowlarkId}`, - traceId, - ); - await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, existingMeadowlarkId); - - // Adding descriptors to outboundRefs for reference checking - const descriptorOutboundRefs = documentInfo.descriptorReferences.map((dr: DocumentReference) => - getMeadowlarkIdForDocumentReference(dr), + }, + { + retries: numberOfRetries, + onRetry: (error, attempt) => { + if (attempt === numberOfRetries) { + Logger.error('Error after maximum retries', error); + } else { + Logger.error('Retrying transaction due to error:', error); + } + }, + }, ); - outboundRefs.push(...descriptorOutboundRefs); - - // Perform insert of references to the references table - // eslint-disable-next-line no-restricted-syntax - for (const ref of outboundRefs) { - Logger.debug( - `${moduleName}.upsertDocument: Inserting reference meadowlarkId ${ref} for document meadowlarkId ${meadowlarkId}`, - ref, - ); - await insertOutboundReferences(client, meadowlarkId, ref as MeadowlarkId); - } - - await commitTransaction(client); - - updateResult = updateDocumentResult - ? { - response: 'UPDATE_SUCCESS', - } - : { - response: 'UPDATE_FAILURE_NOT_EXISTS', - }; + return updateProcess; } catch (e) { await rollbackTransaction(client); Logger.error(`${moduleName}.upsertDocument`, traceId, e); return { response: 'UNKNOWN_FAILURE', failureMessage: e.message }; } - - return updateResult; } diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Upsert.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Upsert.ts index f92946cb..37758a2e 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Upsert.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Upsert.ts @@ -3,6 +3,7 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +import retry from 'async-retry'; import type { PoolClient } from 'pg'; import { UpsertResult, @@ -15,7 +16,7 @@ import { DocumentUuid, MeadowlarkId, } from '@edfi/meadowlark-core'; -import { Logger } from '@edfi/meadowlark-utilities'; +import { Logger, Config } from '@edfi/meadowlark-utilities'; import { deleteOutboundReferencesOfDocumentByMeadowlarkId, insertDocument, @@ -38,127 +39,170 @@ const moduleName = 'postgresql.repository.Upsert'; export async function upsertDocument(upsertRequest: UpsertRequest, client: PoolClient): Promise { const { meadowlarkId, resourceInfo, documentInfo, validateDocumentReferencesExist, traceId } = upsertRequest; Logger.info(`${moduleName}.upsertDocument`, traceId); + let retryCount = 0; - const outboundRefs = documentInfo.documentReferences.map((dr: DocumentReference) => - getMeadowlarkIdForDocumentReference(dr), - ); try { - await beginTransaction(client); - - // Attempt to get the document, to see whether this is an insert or update - const documentFromDb: MeadowlarkDocument = await findDocumentByMeadowlarkId(client, meadowlarkId); - const isInsert: boolean = documentFromDb === NoMeadowlarkDocument; - - // Either get the existing document uuid or create a new one - const documentUuid: DocumentUuid = isInsert ? generateDocumentUuid() : documentFromDb.document_uuid; - - // If an update, check the request for staleness. If request is stale, return conflict. - if (!isInsert && documentFromDb.last_modified_at >= documentInfo.requestTimestamp) { - return { response: 'UPSERT_FAILURE_WRITE_CONFLICT' }; - } - - // If inserting a subclass, check whether the superclass identity is already claimed by a different subclass - if (isInsert && documentInfo.superclassInfo != null) { - const superclassAliasMeadowlarkIdInUseResult: MeadowlarkId[] = await findAliasMeadowlarkId( - client, - getMeadowlarkIdForSuperclassInfo(documentInfo.superclassInfo) as MeadowlarkId, - ); - const superclassAliasMeadowlarkIdInUse: boolean = superclassAliasMeadowlarkIdInUseResult.length !== 0; - - if (superclassAliasMeadowlarkIdInUse) { - Logger.debug( - `${moduleName}.upsertDocument: Upserting document meadowlarkId ${meadowlarkId} failed due to another subclass with the same identity`, - traceId, + const numberOfRetries: number = Config.get('POSTGRES_MAX_NUMBER_OF_RETRIES'); + const upsertProcess = await retry( + async (bail) => { + const outboundRefs = documentInfo.documentReferences.map((dr: DocumentReference) => + getMeadowlarkIdForDocumentReference(dr), ); - - const superclassAliasId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo( - documentInfo.superclassInfo, - ) as MeadowlarkId; - - const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting(client, [ - superclassAliasId, - ]); - - await rollbackTransaction(client); - return { - response: 'INSERT_FAILURE_CONFLICT', - failureMessage: `Insert failed: the identity is in use by '${resourceInfo.resourceName}' which is also a(n) '${documentInfo.superclassInfo.resourceName}'`, - referringDocumentInfo, - }; - } - } - - if (validateDocumentReferencesExist) { - const failures = await validateReferences( - documentInfo.documentReferences, - documentInfo.descriptorReferences, - outboundRefs, - client, - traceId, - ); - // Abort on validation failure - if (failures.length > 0) { - Logger.debug( - `${moduleName}.upsertDocument: Inserting document meadowlarkId ${meadowlarkId} failed due to invalid references`, - traceId, - ); - - const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting(client, [ - meadowlarkId, - ]); - - await rollbackTransaction(client); - return { - response: isInsert ? 'INSERT_FAILURE_REFERENCE' : 'UPDATE_FAILURE_REFERENCE', - failureMessage: { error: { message: 'Reference validation failed', failures } }, - referringDocumentInfo, - }; - } - } - - // Perform the document upsert - Logger.debug(`${moduleName}.upsertDocument: Upserting document meadowlarkId ${meadowlarkId}`, traceId); - - if (isInsert) { - await insertDocument(client, { ...upsertRequest, documentUuid }); - } else { - await updateDocument(client, { ...upsertRequest, documentUuid }); - } - - // Delete existing values from the aliases table - await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId); - - // Perform insert of alias ids - await insertAlias(client, documentUuid, meadowlarkId, meadowlarkId); - if (documentInfo.superclassInfo != null) { - const superclassAliasId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo(documentInfo.superclassInfo) as MeadowlarkId; - await insertAlias(client, documentUuid, meadowlarkId, superclassAliasId); - } - - // Delete existing references in references table - Logger.debug(`${moduleName}.upsertDocument: Deleting references for document meadowlarkId ${meadowlarkId}`, traceId); - await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId); - - // Adding descriptors to outboundRefs for reference checking - const descriptorOutboundRefs = documentInfo.descriptorReferences.map((dr: DocumentReference) => - getMeadowlarkIdForDocumentReference(dr), + try { + await beginTransaction(client); + + // Attempt to get the document, to see whether this is an insert or update + const documentFromDb: MeadowlarkDocument = await findDocumentByMeadowlarkId(client, meadowlarkId); + const isInsert: boolean = documentFromDb === NoMeadowlarkDocument; + + // Either get the existing document uuid or create a new one + const documentUuid: DocumentUuid = isInsert ? generateDocumentUuid() : documentFromDb.document_uuid; + + // If an update, check the request for staleness. If request is stale, return conflict. + if (!isInsert && documentFromDb.last_modified_at >= documentInfo.requestTimestamp) { + return { response: 'UPSERT_FAILURE_WRITE_CONFLICT' }; + } + + // If inserting a subclass, check whether the superclass identity is already claimed by a different subclass + if (isInsert && documentInfo.superclassInfo != null) { + const superclassAliasMeadowlarkIdInUseResult: MeadowlarkId[] = await findAliasMeadowlarkId( + client, + getMeadowlarkIdForSuperclassInfo(documentInfo.superclassInfo) as MeadowlarkId, + ); + const superclassAliasMeadowlarkIdInUse: boolean = superclassAliasMeadowlarkIdInUseResult.length !== 0; + + if (superclassAliasMeadowlarkIdInUse) { + Logger.debug( + `${moduleName}.upsertDocument: Upserting document meadowlarkId ${meadowlarkId} failed due to another subclass with the same identity`, + traceId, + ); + + const superclassAliasId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo( + documentInfo.superclassInfo, + ) as MeadowlarkId; + + const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting( + client, + [superclassAliasId], + ); + + await rollbackTransaction(client); + return { + response: 'INSERT_FAILURE_CONFLICT', + failureMessage: `Insert failed: the identity is in use by '${resourceInfo.resourceName}' which is also a(n) '${documentInfo.superclassInfo.resourceName}'`, + referringDocumentInfo, + }; + } + } + + if (validateDocumentReferencesExist) { + const failures = await validateReferences( + documentInfo.documentReferences, + documentInfo.descriptorReferences, + outboundRefs, + client, + traceId, + ); + // Abort on validation failure + if (failures.length > 0) { + Logger.debug( + `${moduleName}.upsertDocument: Inserting document meadowlarkId ${meadowlarkId} failed due to invalid references`, + traceId, + ); + + const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting( + client, + [meadowlarkId], + ); + + await rollbackTransaction(client); + return { + response: isInsert ? 'INSERT_FAILURE_REFERENCE' : 'UPDATE_FAILURE_REFERENCE', + failureMessage: { error: { message: 'Reference validation failed', failures } }, + referringDocumentInfo, + }; + } + } + + // Perform the document upsert + Logger.debug(`${moduleName}.upsertDocument: Upserting document meadowlarkId ${meadowlarkId}`, traceId); + + if (isInsert) { + await insertDocument(client, { ...upsertRequest, documentUuid }); + } else { + await updateDocument(client, { ...upsertRequest, documentUuid }); + } + + // Delete existing values from the aliases table + await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId); + + // Perform insert of alias ids + await insertAlias(client, documentUuid, meadowlarkId, meadowlarkId); + if (documentInfo.superclassInfo != null) { + const superclassAliasId: MeadowlarkId = getMeadowlarkIdForSuperclassInfo( + documentInfo.superclassInfo, + ) as MeadowlarkId; + await insertAlias(client, documentUuid, meadowlarkId, superclassAliasId); + } + + // Delete existing references in references table + Logger.debug( + `${moduleName}.upsertDocument: Deleting references for document meadowlarkId ${meadowlarkId}`, + traceId, + ); + await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId); + + // Adding descriptors to outboundRefs for reference checking + const descriptorOutboundRefs = documentInfo.descriptorReferences.map((dr: DocumentReference) => + getMeadowlarkIdForDocumentReference(dr), + ); + outboundRefs.push(...descriptorOutboundRefs); + + // Perform insert of references to the references table + // eslint-disable-next-line no-restricted-syntax + for (const ref of outboundRefs) { + Logger.debug( + `post${moduleName}.upsertDocument: Inserting reference meadowlarkId ${ref} for document meadowlarkId ${meadowlarkId}`, + ref, + ); + await insertOutboundReferences(client, meadowlarkId, ref as MeadowlarkId); + } + + await commitTransaction(client); + return isInsert + ? { response: 'INSERT_SUCCESS', newDocumentUuid: documentUuid } + : { response: 'UPDATE_SUCCESS', existingDocumentUuid: documentUuid }; + } catch (error) { + retryCount += 1; + await rollbackTransaction(client); + // If there's a serialization failure, rollback the transaction + if (error.code === '40001') { + if (retryCount >= numberOfRetries) { + throw Error('Error after maximum retries'); + } + // Throws the error to be handled by async-retry + throw error; + } else { + // If it's not a serialization failure, don't retry and rethrow the error + Logger.error(`${moduleName}.upsertDocument`, traceId, error); + // Throws the error to be handled by the caller + bail(error); + } + } + return { response: 'UNKNOWN_FAILURE', failureMessage: '' }; + }, + { + retries: numberOfRetries, + onRetry: (error, attempt) => { + if (attempt === numberOfRetries) { + Logger.error('Error after maximum retries', error); + } else { + Logger.error('Retrying transaction due to error:', error); + } + }, + }, ); - outboundRefs.push(...descriptorOutboundRefs); - - // Perform insert of references to the references table - // eslint-disable-next-line no-restricted-syntax - for (const ref of outboundRefs) { - Logger.debug( - `post${moduleName}.upsertDocument: Inserting reference meadowlarkId ${ref} for document meadowlarkId ${meadowlarkId}`, - ref, - ); - await insertOutboundReferences(client, meadowlarkId, ref as MeadowlarkId); - } - - await commitTransaction(client); - return isInsert - ? { response: 'INSERT_SUCCESS', newDocumentUuid: documentUuid } - : { response: 'UPDATE_SUCCESS', existingDocumentUuid: documentUuid }; + return upsertProcess; } catch (e) { Logger.error(`${moduleName}.upsertDocument`, traceId, e); await rollbackTransaction(client); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/authorization/CreateAuthorizationClient.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/authorization/CreateAuthorizationClient.ts index 5fdc7dd7..dc563937 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/authorization/CreateAuthorizationClient.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/authorization/CreateAuthorizationClient.ts @@ -3,7 +3,8 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. -import { Logger } from '@edfi/meadowlark-utilities'; +import retry from 'async-retry'; +import { Logger, Config } from '@edfi/meadowlark-utilities'; import { CreateAuthorizationClientRequest, CreateAuthorizationClientResult } from '@edfi/meadowlark-authz-server'; import { PoolClient } from 'pg'; import { AuthorizationDocument, authorizationDocumentFromCreate } from '../../model/AuthorizationDocument'; @@ -16,24 +17,57 @@ export async function createAuthorizationClientDocument( client: PoolClient, ): Promise { const createResult: CreateAuthorizationClientResult = { response: 'UNKNOWN_FAILURE' }; + let retryCount = 0; try { - await beginTransaction(client); - const authorizationClient: AuthorizationDocument = authorizationDocumentFromCreate(request); - const hasResults = await insertOrUpdateAuthorization(authorizationClient, client); - if (hasResults) { - createResult.response = 'CREATE_SUCCESS'; - } else { - const msg = 'Error inserting or updating the authorization client in the PostgreSQL database.'; - Logger.error(functionName, request.traceId, msg); - await rollbackTransaction(client); - } - await commitTransaction(client); + const numberOfRetries: number = Config.get('POSTGRES_MAX_NUMBER_OF_RETRIES'); + const createProcess = await retry( + async (bail) => { + try { + await beginTransaction(client); + const authorizationClient: AuthorizationDocument = authorizationDocumentFromCreate(request); + const hasResults = await insertOrUpdateAuthorization(authorizationClient, client); + if (hasResults) { + createResult.response = 'CREATE_SUCCESS'; + } else { + const msg = 'Error inserting or updating the authorization client in the PostgreSQL database.'; + Logger.error(functionName, request.traceId, msg); + await rollbackTransaction(client); + } + await commitTransaction(client); + return createResult; + } catch (error) { + retryCount += 1; + await rollbackTransaction(client); + // If there's a serialization failure, rollback the transaction + if (error.code === '40001') { + if (retryCount >= numberOfRetries) { + throw Error('Error after maximum retries'); + } + // Throws the error to be handled by async-retry + throw error; + } else { + // If it's not a serialization failure, don't retry and rethrow the error + Logger.error(`${functionName}.createAuthorizationClientDocument`, request.traceId, error); + // Throws the error to be handled by the caller + bail(error); + } + } + return createResult; + }, + { + retries: numberOfRetries, + onRetry: (error, attempt) => { + if (attempt === numberOfRetries) { + Logger.error('Error after maximum retries', error); + } else { + Logger.error('Retrying transaction due to error:', error); + } + }, + }, + ); + return createProcess; } catch (e) { - if (client) { - await rollbackTransaction(client); - } Logger.error(functionName, request.traceId, e); } - return createResult; } diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Delete.test.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Delete.test.ts index dd26a48f..d36d34f1 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Delete.test.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Delete.test.ts @@ -24,6 +24,7 @@ import { generateDocumentUuid, UpsertResult, } from '@edfi/meadowlark-core'; +import * as utilities from '@edfi/meadowlark-utilities'; import type { PoolClient } from 'pg'; import { deleteAll, retrieveReferencesByMeadowlarkIdSql } from './TestHelper'; import { getSharedClient, resetSharedClient } from '../../src/repository/Db'; @@ -35,8 +36,11 @@ import { findDocumentByMeadowlarkId, findReferencingMeadowlarkIds, } from '../../src/repository/SqlHelper'; +import * as SqlHelper from '../../src/repository/SqlHelper'; import { MeadowlarkDocument, NoMeadowlarkDocument } from '../../src/model/MeadowlarkDocument'; +jest.setTimeout(70000); + const newUpsertRequest = (): UpsertRequest => ({ meadowlarkId: '' as MeadowlarkId, resourceInfo: NoResourceInfo, @@ -517,3 +521,107 @@ describe('given the delete of a subclass document referenced by an existing docu expect(result.document_identity.schoolId).toBe('123'); }); }); + +describe('given the delete of an existing document with Postgresql SSI', () => { + const retryNumberOfTimes = 2; + let client; + let upsertDocumentUuid; + + const resourceInfo: ResourceInfo = { + ...newResourceInfo(), + resourceName: 'School', + }; + const documentInfo: DocumentInfo = { + ...newDocumentInfo(), + documentIdentity: { natural: 'delete2' }, + }; + const meadowlarkId = meadowlarkIdForDocumentIdentity(resourceInfo, documentInfo.documentIdentity); + + beforeEach(async () => { + client = await getSharedClient(); + const upsertRequest: UpsertRequest = { + ...newUpsertRequest(), + meadowlarkId, + documentInfo, + edfiDoc: { natural: 'key' }, + }; + // insert the initial version + const upsertResult: UpsertResult = await upsertDocument(upsertRequest, client); + upsertDocumentUuid = upsertResult.response === 'INSERT_SUCCESS' ? upsertResult?.newDocumentUuid : ('' as DocumentUuid); + }); + afterEach(async () => { + await deleteAll(client); + client.release(); + await resetSharedClient(); + jest.clearAllMocks(); + }); + + it('should retry on error 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findAliasMeadowlarkIdsForDocumentByDocumentUuid') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + + const deleteResult = await deleteDocumentByDocumentUuid( + { ...newDeleteRequest(), documentUuid: upsertDocumentUuid, resourceInfo }, + client, + ); + expect(deleteResult.response).toBe('DELETE_SUCCESS'); + }); + + it('should not retry on error not equal to 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { code: '50000', message: 'Any exception different of SSI 40001' }; + // Mock selectReferences to throw an error + jest.spyOn(SqlHelper, 'findAliasMeadowlarkIdsForDocumentByDocumentUuid').mockImplementation(() => { + throw mockError; + }); + let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' }; + deleteResult = await deleteDocumentByDocumentUuid( + { ...newDeleteRequest(), documentUuid: upsertDocumentUuid, resourceInfo }, + client, + ); + expect(deleteResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Any exception different of SSI 40001', + }); + }); + + it('should Throw Error When Retry Limit Exceeded', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findAliasMeadowlarkIdsForDocumentByDocumentUuid') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + + let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' }; + deleteResult = await deleteDocumentByDocumentUuid( + { ...newDeleteRequest(), documentUuid: upsertDocumentUuid, resourceInfo }, + client, + ); + expect(deleteResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Error after maximum retries', + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Update.test.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Update.test.ts index 10d31caf..6c5d2f7d 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Update.test.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Update.test.ts @@ -24,6 +24,7 @@ import { UpsertRequest, UpsertResult, } from '@edfi/meadowlark-core'; +import * as utilities from '@edfi/meadowlark-utilities'; import type { PoolClient } from 'pg'; import { resetSharedClient, getSharedClient } from '../../src/repository/Db'; import { updateDocumentByDocumentUuid } from '../../src/repository/Update'; @@ -35,10 +36,11 @@ import { findDocumentByDocumentUuid, findDocumentByMeadowlarkId, } from '../../src/repository/SqlHelper'; +import * as SqlHelper from '../../src/repository/SqlHelper'; import { MeadowlarkDocument } from '../../src/model/MeadowlarkDocument'; +jest.setTimeout(70000); const requestTimestamp: number = 1683326572053; - const documentUuid: DocumentUuid = 'feb82f3e-3685-4868-86cf-f4b91749a799' as DocumentUuid; let resultDocumentUuid: DocumentUuid; @@ -949,3 +951,121 @@ describe('given the attempted update of an existing document with a stale reques expect(result.last_modified_at).toBe(requestTimestamp); }); }); + +describe('given the update of an existing document with Postgresql SSI', () => { + const retryNumberOfTimes = 2; + let client: PoolClient; + let updateResult: UpdateResult; + + const resourceInfo: ResourceInfo = { + ...newResourceInfo(), + resourceName: 'School', + }; + const documentInfoBase: DocumentInfo = { + ...newDocumentInfo(), + documentIdentity: { natural: 'update2' }, + }; + const meadowlarkId = meadowlarkIdForDocumentIdentity(resourceInfo, documentInfoBase.documentIdentity); + let updateRequest: UpdateRequest; + let upsertRequest: UpsertRequest; + + beforeEach(async () => { + client = await getSharedClient(); + upsertRequest = { + ...newUpsertRequest(), + meadowlarkId, + resourceInfo, + documentInfo: { ...documentInfoBase, requestTimestamp }, + edfiDoc: { natural: 'key' }, + }; + updateRequest = { + ...newUpdateRequest(), + documentUuid, + meadowlarkId, + resourceInfo, + documentInfo: { ...documentInfoBase, requestTimestamp: requestTimestamp + 1 }, + edfiDoc: { natural: 'key' }, + }; + // insert the initial version + const upsertResult: UpsertResult = await upsertDocument(upsertRequest, client); + if (upsertResult.response === 'INSERT_SUCCESS') { + resultDocumentUuid = upsertResult.newDocumentUuid; + } else if (upsertResult.response === 'UPDATE_SUCCESS') { + resultDocumentUuid = upsertResult.existingDocumentUuid; + } else { + resultDocumentUuid = '' as DocumentUuid; + } + }); + + afterEach(async () => { + await deleteAll(client); + client.release(); + await resetSharedClient(); + jest.clearAllMocks(); + }); + + it('should retry on error 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findDocumentByDocumentUuid') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + updateResult = await updateDocumentByDocumentUuid( + { ...updateRequest, documentUuid: resultDocumentUuid, edfiDoc: { changeToDoc: true } }, + client, + ); + expect(updateResult.response).toBe('UPDATE_SUCCESS'); + }); + + it('should not retry on error not equal to 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { code: '50000', message: 'Any exception different of SSI 40001' }; + // Mock selectReferences to throw an error + jest.spyOn(SqlHelper, 'findDocumentByDocumentUuid').mockImplementation(() => { + throw mockError; + }); + updateResult = await updateDocumentByDocumentUuid( + { ...updateRequest, documentUuid: resultDocumentUuid, edfiDoc: { changeToDoc: true } }, + client, + ); + expect(updateResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Any exception different of SSI 40001', + }); + }); + + it('should Throw Error When Retry Limit Exceeded', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findDocumentByDocumentUuid') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + updateResult = await updateDocumentByDocumentUuid( + { ...updateRequest, documentUuid: resultDocumentUuid, edfiDoc: { changeToDoc: true } }, + client, + ); + expect(updateResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Error after maximum retries', + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Upsert.test.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Upsert.test.ts index 4b98845f..3ba0b8e0 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Upsert.test.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/Upsert.test.ts @@ -21,15 +21,17 @@ import { MeadowlarkId, TraceId, } from '@edfi/meadowlark-core'; +import * as utilities from '@edfi/meadowlark-utilities'; import type { PoolClient } from 'pg'; import { getSharedClient, resetSharedClient } from '../../src/repository/Db'; import { findDocumentByMeadowlarkId, findAliasMeadowlarkIdsForDocumentByMeadowlarkId } from '../../src/repository/SqlHelper'; import { upsertDocument } from '../../src/repository/Upsert'; import { deleteAll, retrieveReferencesByMeadowlarkIdSql, verifyAliasMeadowlarkId } from './TestHelper'; import { MeadowlarkDocument, NoMeadowlarkDocument } from '../../src/model/MeadowlarkDocument'; +import * as SqlHelper from '../../src/repository/SqlHelper'; +jest.setTimeout(70000); const requestTimestamp: number = 1683326572053; - const newUpsertRequest = (): UpsertRequest => ({ meadowlarkId: '' as MeadowlarkId, resourceInfo: NoResourceInfo, @@ -1014,3 +1016,115 @@ describe('given an update of a subclass document referenced by an existing docum expect(result.last_modified_at).toBe(requestTimestamp + 2); }); }); + +describe('given the upsert of a new document with Postgresql SSI', () => { + let client: PoolClient; + let upsertResult: UpsertResult; + const retryNumberOfTimes = 2; + + const resourceInfo: ResourceInfo = { + ...newResourceInfo(), + resourceName: 'School', + }; + const documentInfo: DocumentInfo = { + ...newDocumentInfo(), + documentIdentity: { natural: 'upsert1' }, + }; + const meadowlarkId = meadowlarkIdForDocumentIdentity(resourceInfo, documentInfo.documentIdentity); + + beforeEach(async () => { + client = await getSharedClient(); + }); + + afterEach(async () => { + await deleteAll(client); + client.release(); + await resetSharedClient(); + }); + + it('should retry on error 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findDocumentByMeadowlarkId') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + upsertResult = await upsertDocument( + { + ...newUpsertRequest(), + meadowlarkId, + resourceInfo, + documentInfo, + edfiDoc: { call: 'one' }, + validateDocumentReferencesExist: false, + }, + client, + ); + expect(upsertResult.response).toBe('INSERT_SUCCESS'); + }); + + it('should not retry on error not equal to 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { code: '50000', message: 'Any exception different of SSI 40001' }; + // Mock selectReferences to throw an error + jest.spyOn(SqlHelper, 'findDocumentByMeadowlarkId').mockImplementation(() => { + throw mockError; + }); + upsertResult = await upsertDocument( + { + ...newUpsertRequest(), + meadowlarkId, + resourceInfo, + documentInfo, + edfiDoc: { call: 'one' }, + validateDocumentReferencesExist: false, + }, + client, + ); + expect(upsertResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Any exception different of SSI 40001', + }); + }); + + it('should Throw Error When Retry Limit Exceeded', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'findDocumentByMeadowlarkId') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + upsertResult = await upsertDocument( + { + ...newUpsertRequest(), + meadowlarkId, + resourceInfo, + documentInfo, + edfiDoc: { call: 'one' }, + validateDocumentReferencesExist: false, + }, + client, + ); + expect(upsertResult).toMatchObject({ + response: 'UNKNOWN_FAILURE', + failureMessage: 'Error after maximum retries', + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/authorization/CreateAuthorizationClient.test.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/authorization/CreateAuthorizationClient.test.ts index 73a75476..4f60b967 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/authorization/CreateAuthorizationClient.test.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/authorization/CreateAuthorizationClient.test.ts @@ -4,14 +4,16 @@ // See the LICENSE and NOTICES files in the project root for more information. import { CreateAuthorizationClientRequest, CreateAuthorizationClientResult } from '@edfi/meadowlark-authz-server'; +import * as utilities from '@edfi/meadowlark-utilities'; import type { PoolClient } from 'pg'; import { getSharedClient, resetSharedClient } from '../../../src/repository/Db'; import { createAuthorizationClientDocument } from '../../../src/repository/authorization/CreateAuthorizationClient'; import { deleteAllAuthorizations } from '../TestHelper'; import { getAuthorizationClientDocumentById } from '../../../src/repository/SqlHelper'; +import * as SqlHelper from '../../../src/repository/SqlHelper'; const clientId = 'clientId'; - +jest.setTimeout(70000); const newCreateAuthorizationClientRequest = (): CreateAuthorizationClientRequest => ({ clientId, clientSecretHashed: 'clientSecretHashed', @@ -62,3 +64,70 @@ describe('given the create of a new authorization client', () => { `); }); }); + +describe('given the create of a new authorization client with Postgresql SSI', () => { + let client: PoolClient; + let createClientRequest: CreateAuthorizationClientResult; + const retryNumberOfTimes = 2; + + beforeEach(async () => { + client = await getSharedClient(); + }); + + afterEach(async () => { + if (client) { + await deleteAllAuthorizations(client); + client.release(); + await resetSharedClient(); + } + }); + it('should retry on error 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'insertOrUpdateAuthorization') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + createClientRequest = await createAuthorizationClientDocument(newCreateAuthorizationClientRequest(), client); + expect(createClientRequest.response).toBe('CREATE_SUCCESS'); + }); + + it('should not retry on error not equal to 40001', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { code: '50000', message: 'Any exception different of SSI 40001' }; + // Mock selectReferences to throw an error + jest.spyOn(SqlHelper, 'insertOrUpdateAuthorization').mockImplementation(() => { + throw mockError; + }); + createClientRequest = await createAuthorizationClientDocument(newCreateAuthorizationClientRequest(), client); + expect(createClientRequest.response).toBe('UNKNOWN_FAILURE'); + }); + + it('should Throw Error When Retry Limit Exceeded', async () => { + jest.spyOn(utilities.Config, 'get').mockImplementationOnce(() => retryNumberOfTimes); + const mockError = { + code: '40001', + message: 'Could not serialize access due to read/write dependencies among transactions', + }; + jest + .spyOn(SqlHelper, 'insertOrUpdateAuthorization') + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }) + .mockImplementationOnce(() => { + throw mockError; + }); + createClientRequest = await createAuthorizationClientDocument(newCreateAuthorizationClientRequest(), client); + expect(createClientRequest.response).toBe('UNKNOWN_FAILURE'); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/locking/Delete.test.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/locking/Delete.test.ts index c6adf7ac..0c8452be 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/locking/Delete.test.ts +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/test/integration/locking/Delete.test.ts @@ -36,6 +36,7 @@ import { upsertDocument } from '../../../src/repository/Upsert'; import { deleteAll } from '../TestHelper'; import { MeadowlarkDocument, NoMeadowlarkDocument } from '../../../src/model/MeadowlarkDocument'; +jest.setTimeout(90000); // A bunch of setup stuff const newUpsertRequest = (): UpsertRequest => ({ meadowlarkId: '' as MeadowlarkId, diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index 1e934fac..4af3eac8 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -111,6 +111,7 @@ "@edfi/meadowlark-authz-server": "^v0.3.6-pre-36", "@edfi/meadowlark-core": "^v0.3.6-pre-36", "@edfi/meadowlark-utilities": "^v0.3.6-pre-36", + "async-retry": "^1.3.3", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" @@ -6201,8 +6202,7 @@ "node_modules/async-retry": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha1-Dn82wE2EeOeli9vtgM7fl3eF8oA=", - "license": "MIT", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", "dependencies": { "retry": "0.13.1" } diff --git a/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts b/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts index fcbce97b..294b77a0 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts +++ b/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts @@ -53,7 +53,8 @@ export type ConfigKeys = | 'FASTIFY_RATE_LIMIT' | 'HTTP_PROTOCOL_AND_SERVER' | 'DISABLE_LOG_ANONYMIZATION' - | 'MONGODB_MAX_NUMBER_OF_RETRIES'; + | 'MONGODB_MAX_NUMBER_OF_RETRIES' + | 'POSTGRES_MAX_NUMBER_OF_RETRIES'; const ThrowIfNotFound = undefined; const CpuCount = os.cpus().length; @@ -178,4 +179,5 @@ export async function initializeConfig(provider: ConfigPlugin) { set('BEGIN_ALLOWED_SCHOOL_YEAR', await provider.getInt('BEGIN_ALLOWED_SCHOOL_YEAR', 1900)); set('END_ALLOWED_SCHOOL_YEAR', await provider.getInt('END_ALLOWED_SCHOOL_YEAR', 2100)); set('MONGODB_MAX_NUMBER_OF_RETRIES', await provider.getInt('MONGODB_MAX_NUMBER_OF_RETRIES', 1)); + set('POSTGRES_MAX_NUMBER_OF_RETRIES', await provider.getInt('POSTGRES_MAX_NUMBER_OF_RETRIES', 1)); } diff --git a/docs/performance-testing/postgres-SSI-performance-testing.md b/docs/performance-testing/postgres-SSI-performance-testing.md new file mode 100644 index 00000000..41d16e20 --- /dev/null +++ b/docs/performance-testing/postgres-SSI-performance-testing.md @@ -0,0 +1,57 @@ +# RND-643: PostgreSQL Serializable Snapshot Isolation (SSI) performance testing + +## Goal + +Run Postgresql performance to compare the performance of the SSI implementation vs the SELECT...FOR UPDATE/SHARE implementation. + +## Description + +To test performance we use Autocannon tool. + +Changed transaction usage to use SSI. Additionally, retry logic was added with this change. With this logic, if a 40001 error occurs, a retry is executed. All functions that use transactions have been modified. + +As a side effect, the crash test\delete.test.ts failed. This test has transactions to create a lock and has no retry mechanism, so the test creates a deadlock and returns a timeout error. + +To capture database statistics we are reading data from pg_stat_database. A process runs each 5 seconds and inserts a row with the current statistics. With all the rows inserted, we can get an average of some indicators. From pg_stat_database we can get: + +- xact_commit: number of transactions in the database that have been completed successfully. +- xact_rollback: number of transactions that have been cancelled or rolled back. +- blks_read: number of disk blocks that have been read directly from the disk +- blks_hit: This is the number of times that the needed data was found in the cache or buffer. +- tup_returned: number of rows returned by queries in the database +- tup_fetched: number of rows fetched by queries in the database +- tup_inserted: number of rows inserted +- tup_updated: number of rows updated +- tup_deleted: number of rows deleted + +## Postgres Comparison + +For testing, Autocannon was run in the existing version in main and in the modified version. For each test, 3 iterations were executed and the comparison was made with the data obtained. + +First, we tested Autocannon with the original version and the modified version. + +||2xxx|Non-2xxx|Latency|Request|Throughput +| :--- | ---: | ---: | ---: | ---: | ---: | +**Non-SSI**|7075.67|0|564.35|44.51|15532.14 +**SSI**|5723.33|8659.66666666667|277.82|90.28|28020.23 +**Differences**|**-19.11%**|**100.00%**|**-50.77%**|**102.83%**|**80.40%** + +For this case we can see: + +- The SSI version has less 2xxx responses and more Non-2xxx. Now, we this change we are not locking the database, so that behavior is expected. +- In the same case, the SSI version has less latency and more requests and throughput compared to the original version. + +With this results we can see some improvement in the performance. + +Now, if we compare database statistics, we have the following results: + +|Pool Size | Avg commit | Avg rollback | Avg disk blocks read | Avg blocks hit cache | Avg tuples returned | Avg tuples fetched | Avg tuples_inserted | Avg tuples updated | Avg tuples deleted | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +Non-SSI|328.20|0|30.52|7662.52|1432.53|788.24|327.96|73.20|281.24 +SSI|149.51|98.88|17.03|5955.38|4748.24|445.19|151.52|46.46|153.13 +|**Differences**|**-54.00%**|**100.00%**|**-44.00%**|**-22.00%**|**231.00%**|**-44.00%**|**-54.00%**|**-37.00%**|**-46.00%** + +These results are consistent with the Autocannon results: + +- SSI version has less commits than previous version, and more rollbacks than previous version, that makes sense because we are not locking the database and we are executing a rollback to retry. +- Also, SSI version returned more tuples but affected tuples are less compared to previous version. That behavior is consistent with a retry mechanism, because when we need to retry, we need to restart the process.