Skip to content

Commit

Permalink
[RND-401] Adds retry logic for MongoDB (#216)
Browse files Browse the repository at this point in the history 
* Adds retry logic for MongoDB

* Fixes unit test.

* [RND-401] refactor, update messages

* Changes error messages thrown from the repositories.

* After rebase changes.

* Changes env value name. Adds some documentation.

* Removes unnecessary double question mark. Changes main .env.example file. Changes on Configuration.md file.

---------

Co-authored-by: Brad Banister <[email protected]>
Co-authored-by: Stephen Fuqua <[email protected]>
  • Loading branch information
@DavidJGapCR @bradbanister @stephenfuqua
3 people authored Mar 13, 2023
1 parent 0366d69 commit 4dc877e
Show file tree
Hide file tree
Showing 20 changed files with 620 additions and 70 deletions.
8 changes: 5 additions & 3 deletions Meadowlark-js/.env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ MEADOWLARK_DATABASE_NAME=meadowlark
POSTGRES_HOST=localhost
POSTGRES_PORT=5432

# Do not override MongoDB settings here, because the integration tests use an "in memory"
# version of MongoDB that is configured automatically. Any settings applied here will
# cause the connection to fail.
# With the exception of MONGODB_MAX_NUMBER_OF_RETRIES, do not override MongoDB settings here,
# because the integration tests use an "in memory" # version of MongoDB that is configured automatically.
# Any settings applied here will cause the connection to fail.

MONGODB_MAX_NUMBER_OF_RETRIES=1
1 change: 1 addition & 0 deletions Meadowlark-js/backends/meadowlark-mongodb-backend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
"@edfi/meadowlark-authz-server": "0.3.0-dev.0",
"@edfi/meadowlark-core": "0.3.0-dev.0",
"@edfi/meadowlark-utilities": "0.3.0-dev.0",
"async-retry": "^1.3.3",
"mongodb": "^4.11.0",
"ramda": "0.27.2",
"ts-invariant": "^0.10.3"
Expand Down
58 changes: 43 additions & 15 deletions Meadowlark-js/backends/meadowlark-mongodb-backend/src/repository/Delete.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@

/* eslint-disable no-underscore-dangle */

import { Logger } from '@edfi/meadowlark-utilities';
import { Logger, Config } from '@edfi/meadowlark-utilities';
import { DeleteResult, DeleteRequest, BlockingDocument, DocumentUuid, TraceId } from '@edfi/meadowlark-core';
import { ClientSession, Collection, MongoClient, WithId } from 'mongodb';
import retry from 'async-retry';
import { MeadowlarkDocument } from '../model/MeadowlarkDocument';
import { getDocumentCollection, limitFive, onlyReturnId } from './Db';
import { onlyReturnAliasIds, onlyDocumentsReferencing } from './ReferenceValidation';

const moduleName: string = 'mongodb.repository.Delete';

/**
* Checks for any existing references to the document. If found, provide a DeleteResult with information on the errors.
*
Expand Down Expand Up @@ -43,7 +46,7 @@ async function checkForReferencesToDocument(

// Abort on validation failure
Logger.debug(
`mongodb.repository.Delete.checkForReferencesToDocument: Deleting document uuid ${documentUuid} failed due to existing references`,
`${moduleName}.checkForReferencesToDocument: Deleting document uuid ${documentUuid} failed due to existing references`,
traceId,
);

Expand Down Expand Up @@ -84,17 +87,14 @@ export async function deleteDocumentByIdTransaction(
}

// Perform the document delete
Logger.debug(
`mongodb.repository.Delete.deleteDocumentByIdTransaction: Deleting document documentUuid ${documentUuid}`,
traceId,
);
Logger.debug(`${moduleName}.deleteDocumentByIdTransaction: Deleting document documentUuid ${documentUuid}`, traceId);

const { acknowledged, deletedCount } = await mongoCollection.deleteOne({ documentUuid }, { session });

if (!acknowledged) {
const msg =
'mongoCollection.deleteOne returned acknowledged: false, indicating a problem with write concern configuration';
Logger.error('mongodb.repository.Delete.deleteDocumentByIdTransaction', traceId, msg);
Logger.error(`${moduleName}.deleteDocumentByIdTransaction`, traceId, msg);
return { response: 'UNKNOWN_FAILURE', failureMessage: '' };
}

Expand All @@ -112,16 +112,44 @@ export async function deleteDocumentById(deleteRequest: DeleteRequest, client: M
let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' };
try {
const mongoCollection: Collection<MeadowlarkDocument> = getDocumentCollection(client);
await session.withTransaction(async () => {
deleteResult = await deleteDocumentByIdTransaction(deleteRequest, mongoCollection, session);
if (deleteResult.response !== 'DELETE_SUCCESS') {
await session.abortTransaction();
}
});

const numberOfRetries: number = Config.get('MONGODB_MAX_NUMBER_OF_RETRIES');

await retry(
async () => {
await session.withTransaction(async () => {
deleteResult = await deleteDocumentByIdTransaction(deleteRequest, mongoCollection, session);
if (deleteResult.response !== 'DELETE_SUCCESS') {
await session.abortTransaction();
}
});
},
{
retries: numberOfRetries,
onRetry: () => {
Logger.warn(
`${moduleName}.deleteDocumentById got write conflict error for documentUuid ${deleteRequest.documentUuid}. Retrying...`,
deleteRequest.traceId,
);
},
},
);
} catch (e) {
Logger.error('mongodb.repository.Delete.deleteDocumentById', deleteRequest.traceId, e);
Logger.error(`${moduleName}.deleteDocumentById`, deleteRequest.traceId, e);

let response: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: e.message };

// If this is a MongoError, it has a codeName
if (e.codeName === 'WriteConflict') {
response = {
response: 'DELETE_FAILURE_WRITE_CONFLICT',
failureMessage: 'Write conflict due to concurrent access to this or related resources',
};
}

await session.abortTransaction();
return { response: 'UNKNOWN_FAILURE', failureMessage: e.message };

return response;
} finally {
await session.endSession();
}
Expand Down
112 changes: 68 additions & 44 deletions Meadowlark-js/backends/meadowlark-mongodb-backend/src/repository/Update.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
/* eslint-disable no-underscore-dangle */

import { UpdateResult, UpdateRequest, BlockingDocument } from '@edfi/meadowlark-core';
import { Logger } from '@edfi/meadowlark-utilities';
import { Logger, Config } from '@edfi/meadowlark-utilities';
import { Collection, ClientSession, MongoClient, WithId } from 'mongodb';
import retry from 'async-retry';
import { MeadowlarkDocument, meadowlarkDocumentFrom } from '../model/MeadowlarkDocument';
import { getDocumentCollection, limitFive, onlyReturnId, writeLockReferencedDocuments } from './Db';
import { deleteDocumentByIdTransaction } from './Delete';
Expand Down Expand Up @@ -254,64 +255,87 @@ async function checkForInvalidReferences(
};
}

async function updateDocumentByIdTransaction(
updateRequest: UpdateRequest,
mongoCollection: Collection<MeadowlarkDocument>,
session: ClientSession,
): Promise<UpdateResult> {
const { meadowlarkId, documentUuid, resourceInfo, documentInfo, edfiDoc, validateDocumentReferencesExist, security } =
updateRequest;

if (validateDocumentReferencesExist) {
const invalidReferenceResult: UpdateResult | null = await checkForInvalidReferences(
updateRequest,
mongoCollection,
session,
);
if (invalidReferenceResult !== null) {
return invalidReferenceResult;
}
}

const document: MeadowlarkDocument = meadowlarkDocumentFrom(
resourceInfo,
documentInfo,
documentUuid,
meadowlarkId,
edfiDoc,
validateDocumentReferencesExist,
security.clientId,
);

if (resourceInfo.allowIdentityUpdates) {
return updateAllowingIdentityChange(document, updateRequest, mongoCollection, session);
}
return updateDisallowingIdentityChange(document, updateRequest, mongoCollection, session);
}

/**
* Takes an UpdateRequest and MongoClient from the BackendFacade and performs an update by documentUuid
* and returns the UpdateResult.
*/
export async function updateDocumentById(updateRequest: UpdateRequest, client: MongoClient): Promise<UpdateResult> {
const {
meadowlarkId,
documentUuid,
resourceInfo,
documentInfo,
edfiDoc,
validateDocumentReferencesExist,
traceId,
security,
} = updateRequest;
const { documentUuid, traceId } = updateRequest;
Logger.info(`${moduleName}.updateDocumentById ${documentUuid}`, traceId);

const mongoCollection: Collection<MeadowlarkDocument> = getDocumentCollection(client);
const session: ClientSession = client.startSession();
let updateResult: UpdateResult = { response: 'UNKNOWN_FAILURE' };

try {
await session.withTransaction(async () => {
if (validateDocumentReferencesExist) {
const invalidReferenceResult: UpdateResult | null = await checkForInvalidReferences(
updateRequest,
mongoCollection,
session,
);
if (invalidReferenceResult !== null) {
updateResult = invalidReferenceResult;
return; // exit transaction block
}
}

const document: MeadowlarkDocument = meadowlarkDocumentFrom(
resourceInfo,
documentInfo,
documentUuid,
meadowlarkId,
edfiDoc,
validateDocumentReferencesExist,
security.clientId,
);

if (resourceInfo.allowIdentityUpdates) {
updateResult = await updateAllowingIdentityChange(document, updateRequest, mongoCollection, session);
} else {
updateResult = await updateDisallowingIdentityChange(document, updateRequest, mongoCollection, session);
}

if (updateResult.response !== 'UPDATE_SUCCESS') {
await session.abortTransaction();
}
});
const numberOfRetries: number = Config.get('MONGODB_MAX_NUMBER_OF_RETRIES');

await retry(
async () => {
await session.withTransaction(async () => {
updateResult = await updateDocumentByIdTransaction(updateRequest, mongoCollection, session);
if (updateResult.response !== 'UPDATE_SUCCESS') {
await session.abortTransaction();
}
});
},
{
retries: numberOfRetries,
onRetry: () => {
Logger.warn(
`${moduleName}.updateDocumentById got write conflict error for documentUuid ${updateRequest.documentUuid}. Retrying...`,
updateRequest.traceId,
);
},
},
);
} catch (e) {
Logger.error(`${moduleName}.updateDocumentById`, traceId, e);
await session.abortTransaction();

// If this is a MongoError, it has a codeName
if (e.codeName === 'WriteConflict') {
return {
response: 'UPDATE_FAILURE_WRITE_CONFLICT',
failureMessage: 'Write conflict due to concurrent access to this or related resources.',
};
}

return { response: 'UNKNOWN_FAILURE', failureMessage: e.message };
} finally {
await session.endSession();
Expand Down
39 changes: 32 additions & 7 deletions Meadowlark-js/backends/meadowlark-mongodb-backend/src/repository/Upsert.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import {
DocumentUuid,
generateDocumentUuid,
} from '@edfi/meadowlark-core';
import { Logger } from '@edfi/meadowlark-utilities';
import { Logger, Config } from '@edfi/meadowlark-utilities';
import retry from 'async-retry';
import { MeadowlarkDocument, meadowlarkDocumentFrom } from '../model/MeadowlarkDocument';
import { writeLockReferencedDocuments, asUpsert, limitFive, getDocumentCollection, onlyReturnDocumentUuid } from './Db';
import { onlyDocumentsReferencing, validateReferences } from './ReferenceValidation';
Expand Down Expand Up @@ -157,15 +158,39 @@ export async function upsertDocument(upsertRequest: UpsertRequest, client: Mongo
const session: ClientSession = client.startSession();
let upsertResult: UpsertResult = { response: 'UNKNOWN_FAILURE' };
try {
await session.withTransaction(async () => {
upsertResult = await upsertDocumentTransaction(upsertRequest, mongoCollection, session);
if (upsertResult.response !== 'UPDATE_SUCCESS' && upsertResult.response !== 'INSERT_SUCCESS') {
await session.abortTransaction();
}
});
const numberOfRetries: number = Config.get('MONGODB_MAX_NUMBER_OF_RETRIES');

await retry(
async () => {
await session.withTransaction(async () => {
upsertResult = await upsertDocumentTransaction(upsertRequest, mongoCollection, session);
if (upsertResult.response !== 'UPDATE_SUCCESS' && upsertResult.response !== 'INSERT_SUCCESS') {
await session.abortTransaction();
}
});
},
{
retries: numberOfRetries,
onRetry: () => {
Logger.warn(
`${moduleName}.upsertDocument got write conflict error for meadowlarkId ${upsertRequest.meadowlarkId}. Retrying...`,
upsertRequest.traceId,
);
},
},
);
} catch (e) {
Logger.error(`${moduleName}.upsertDocument`, upsertRequest.traceId, e);
await session.abortTransaction();

// If this is a MongoError, it has a codeName
if (e.codeName === 'WriteConflict') {
return {
response: 'UPSERT_FAILURE_WRITE_CONFLICT',
failureMessage: 'Write conflict due to concurrent access to this or related resources',
};
}

return { response: 'UNKNOWN_FAILURE', failureMessage: e.message };
} finally {
await session.endSession();
Expand Down
Loading

0 comments on commit 4dc877e

Please sign in to comment.