Skip to content

Commit

Permalink
[BIA-563] PostgreSQL backend SqlHelper functions should directly retu…
Browse files Browse the repository at this point in the history
…rn typed results rather than strings (#258)

* [BIA-563] PostgreSQL backend SqlHelper functions should directly return typed results rather than strings

Update sqlHelper to execute sql queries and return an object with the result.
Update references to use the new result instead of execute the sql string.
Update tests.

* Update function to return an array of blockingDocument

* Replace BlockingDocument by ReferringDocumentInfo

* Update validations

* Update transaction

* Remove validation

* Update insert, update and delete to execute sql from sql helper

* Update sqlHelper to avoid returning sql scripts as string.

* Rename MeadowlarkAlias to MeadowlarkDocumentAndAliasId

* Update SqlHelper

Update function name to remove execute prefix.
Update internal documentation
  • Loading branch information
jleiva-gap authored Jun 19, 2023
1 parent 821ba85 commit 5e5e469
Show file tree
Hide file tree
Showing 30 changed files with 703 additions and 466 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
/* eslint-disable no-underscore-dangle */

import { Logger, Config } from '@edfi/meadowlark-utilities';
import { DeleteResult, DeleteRequest, BlockingDocument, DocumentUuid, TraceId } from '@edfi/meadowlark-core';
import { DeleteResult, DeleteRequest, ReferringDocumentInfo, DocumentUuid, TraceId } from '@edfi/meadowlark-core';
import { ClientSession, Collection, MongoClient, WithId } from 'mongodb';
import retry from 'async-retry';
import { MeadowlarkDocument } from '../model/MeadowlarkDocument';
Expand Down Expand Up @@ -55,15 +55,15 @@ async function checkForReferencesToDocument(
.find(onlyDocumentsReferencing(deleteCandidate.aliasMeadowlarkIds), limitFive(session))
.toArray();

const blockingDocuments: BlockingDocument[] = referringDocuments.map((document) => ({
const referringDocumentInfo: ReferringDocumentInfo[] = referringDocuments.map((document) => ({
documentUuid: document.documentUuid,
meadowlarkId: document._id,
resourceName: document.resourceName,
projectName: document.projectName,
resourceVersion: document.resourceVersion,
}));

return { response: 'DELETE_FAILURE_REFERENCE', blockingDocuments };
return { response: 'DELETE_FAILURE_REFERENCE', referringDocumentInfo };
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

/* eslint-disable no-underscore-dangle */

import { UpdateResult, UpdateRequest, BlockingDocument } from '@edfi/meadowlark-core';
import { UpdateResult, UpdateRequest, ReferringDocumentInfo } from '@edfi/meadowlark-core';
import { Logger, Config } from '@edfi/meadowlark-utilities';
import { Collection, ClientSession, MongoClient, WithId } from 'mongodb';
import retry from 'async-retry';
Expand Down Expand Up @@ -62,17 +62,17 @@ async function insertUpdatedDocument(
traceId,
'Got "INSERT_FAILURE_REFERENCE" from upsertDocumentTransaction() but references should not have been validated',
);
return { response: 'UPDATE_FAILURE_REFERENCE', blockingDocuments: upsertResult.blockingDocuments };
return { response: 'UPDATE_FAILURE_REFERENCE', referringDocumentInfo: upsertResult.referringDocumentInfo };
case 'UPDATE_FAILURE_REFERENCE':
// Something unexpected happened. There should have been a prior delete, and validation should not have occurred.
Logger.error(
`${moduleName}.insertUpdatedDocument`,
traceId,
'Got "UPDATE_FAILURE_REFERENCE" from upsertDocumentTransaction() but document should have been deleted first and references should not have been validated',
);
return { response: 'UPDATE_FAILURE_REFERENCE', blockingDocuments: upsertResult.blockingDocuments };
return { response: 'UPDATE_FAILURE_REFERENCE', referringDocumentInfo: upsertResult.referringDocumentInfo };
case 'INSERT_FAILURE_CONFLICT':
return { response: 'UPDATE_FAILURE_CONFLICT', blockingDocuments: upsertResult.blockingDocuments };
return { response: 'UPDATE_FAILURE_CONFLICT', referringDocumentInfo: upsertResult.referringDocumentInfo };
default:
return { response: 'UNKNOWN_FAILURE', failureMessage: upsertResult.failureMessage };
}
Expand Down Expand Up @@ -261,7 +261,7 @@ async function checkForInvalidReferences(
.find(onlyDocumentsReferencing([meadowlarkId]), limitFive(session))
.toArray();

const blockingDocuments: BlockingDocument[] = referringDocuments.map((document) => ({
const referringDocumentInfo: ReferringDocumentInfo[] = referringDocuments.map((document) => ({
documentUuid: document.documentUuid,
meadowlarkId: document._id,
resourceName: document.resourceName,
Expand All @@ -272,7 +272,7 @@ async function checkForInvalidReferences(
return {
response: 'UPDATE_FAILURE_REFERENCE',
failureMessage: { message: 'Reference validation failed', failures },
blockingDocuments,
referringDocumentInfo,
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
UpsertResult,
UpsertRequest,
getMeadowlarkIdForSuperclassInfo,
BlockingDocument,
ReferringDocumentInfo,
DocumentUuid,
generateDocumentUuid,
MeadowlarkId,
Expand Down Expand Up @@ -63,7 +63,7 @@ export async function upsertDocumentTransaction(
return {
response: 'INSERT_FAILURE_CONFLICT',
failureMessage: `Insert failed: the identity is in use by '${resourceInfo.resourceName}' which is also a(n) '${documentInfo.superclassInfo.resourceName}'`,
blockingDocuments: [
referringDocumentInfo: [
{
documentUuid: superclassAliasMeadowlarkIdInUse.documentUuid,
meadowlarkId: superclassAliasMeadowlarkIdInUse._id,
Expand Down Expand Up @@ -96,7 +96,7 @@ export async function upsertDocumentTransaction(
.find(onlyDocumentsReferencing([meadowlarkId]), limitFive(session))
.toArray();

const blockingDocuments: BlockingDocument[] = referringDocuments.map((document) => ({
const referringDocumentInfo: ReferringDocumentInfo[] = referringDocuments.map((document) => ({
documentUuid: document.documentUuid,
meadowlarkId: document._id,
resourceName: document.resourceName,
Expand All @@ -107,7 +107,7 @@ export async function upsertDocumentTransaction(
return {
response: isInsert ? 'INSERT_FAILURE_REFERENCE' : 'UPDATE_FAILURE_REFERENCE',
failureMessage: { error: { message: 'Reference validation failed', failures } },
blockingDocuments,
referringDocumentInfo,
};
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// SPDX-License-Identifier: Apache-2.0
// Licensed to the Ed-Fi Alliance under one or more agreements.
// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
// See the LICENSE and NOTICES files in the project root for more information.

import { DocumentIdentity, MeadowlarkId, NoDocumentIdentity, DocumentUuid } from '@edfi/meadowlark-core';

export interface MeadowlarkDocumentId {
/**
* A string hash derived from the project name, resource name
* and identity of the API document. This field replaces the built-in MongoDB _id.
*/
meadowlark_id: MeadowlarkId;
/**
* The UUID for the document.
*/
document_uuid: DocumentUuid;
}

export interface MeadowlarkDocument extends MeadowlarkDocumentId {
/**
* The identity elements extracted from the API document.
*/
document_identity: DocumentIdentity;

/**
* The MetaEd project name the API document resource is defined in e.g. "EdFi" for a data standard entity.
*/
project_name: string;

/**
* The name of the resource. Typically, this is the same as the corresponding MetaEd entity name. However,
* there are exceptions, for example descriptors have a "Descriptor" suffix on their resource name.
*/
resource_name: string;

/**
* The resource version as a string. This is the same as the MetaEd project version
* the entity is defined in e.g. "3.3.1-b" for a 3.3b data standard entity.
*/
resource_version: string;

/**
* The Ed-Fi ODS/API document itself.
*/
edfi_doc: any;

/**
* True if this document has been reference and descriptor validated.
*/
validated: boolean;

/**
* True if this document is a descriptor.
*/
is_descriptor: boolean;

/**
* Creator of this document
*/
created_by: string;
}

/**
* Creates a new empty newMeadowlarkDocument object
*/
export function newMeadowlarkDocument(): MeadowlarkDocument {
return {
meadowlark_id: undefined as unknown as MeadowlarkId,
document_uuid: undefined as unknown as DocumentUuid,
document_identity: NoDocumentIdentity,
project_name: '',
resource_name: '',
resource_version: '',
edfi_doc: null,
validated: false,
is_descriptor: false,
created_by: '',
};
}

/**
* The null object for DocumentInfo
*/
export const NoMeadowlarkDocument = Object.freeze({
...newMeadowlarkDocument(),
});

export function isMeadowlarkDocumentEmpty(meadowlarkDocument: MeadowlarkDocument): boolean {
const noMeadowlarkDocument = NoMeadowlarkDocument;
return meadowlarkDocument === noMeadowlarkDocument;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// Licensed to the Ed-Fi Alliance under one or more agreements.
// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
// See the LICENSE and NOTICES files in the project root for more information.

import { MeadowlarkId } from '@edfi/meadowlark-core';
import { MeadowlarkDocumentId } from './MeadowlarkDocument';

export interface MeadowlarkDocumentAndAliasId extends MeadowlarkDocumentId {
/**
* Alias meadowlarkId.
*/
alias_meadowlark_id: MeadowlarkId;
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,7 @@
import { Pool, Client } from 'pg';
import type { PoolClient } from 'pg';
import { Config, Logger } from '@edfi/meadowlark-utilities';
import {
createDocumentTableSql,
createDocumentTableUniqueIndexSql,
createReferencesTableCheckingIndexSql,
createReferencesTableDeletingIndexSql,
createReferencesTableSql,
createSchemaSql,
createDatabaseSql,
createAliasesTableSql,
createAliasesTableDocumentIndexSql,
createAliasesTableAliasIndexSql,
} from './SqlHelper';
import { createDatabase, checkExistsAndCreateTables } from './SqlHelper';

let singletonDbPool: Pool | null = null;

Expand All @@ -31,27 +20,6 @@ const getDbConfiguration = () => ({

const moduleName = 'postgresql.repository.Db';

/**
* Checks that the meadowlark schema, document and references tables exist in the database, if not will create them
* @param client The Postgres client for querying
*/
export async function checkExistsAndCreateTables(client: PoolClient) {
try {
await client.query(createSchemaSql);
await client.query(createDocumentTableSql);
await client.query(createDocumentTableUniqueIndexSql);
await client.query(createReferencesTableSql);
await client.query(createReferencesTableCheckingIndexSql);
await client.query(createReferencesTableDeletingIndexSql);
await client.query(createAliasesTableSql);
await client.query(createAliasesTableDocumentIndexSql);
await client.query(createAliasesTableAliasIndexSql);
} catch (e) {
Logger.error(`${moduleName}.checkExistsAndCreateTables error connecting to PostgreSQL`, null, e);
throw e;
}
}

/**
* Create a connection pool, check that the database and table structure is in place
* or create it if not and then return the pool
Expand Down Expand Up @@ -92,7 +60,7 @@ export async function createConnectionPoolAndReturnClient(): Promise<PoolClient>
const client: Client = new Client(getDbConfiguration());
try {
await client.connect();
await client.query(createDatabaseSql(meadowlarkDbName));
await createDatabase(client, meadowlarkDbName);

Logger.info(`${moduleName}.createConnectionPoolAndReturnClient database ${meadowlarkDbName} created successfully`, null);
} catch (e) {
Expand Down
Loading

0 comments on commit 5e5e469

Please sign in to comment.