Perf/improve email delivery (erxes#2535)

Email delivery related improvements

.gitignore

@@ -22,3 +22,5 @@ gitinfo.json
 ecosystem.config.js
 nginx.conf
 private
+engages-email-sender/src/__tests__/coverage/
+*.js.map

api/src/__tests__/engageMessageQueries.test.ts

@@ -2,6 +2,7 @@ import * as sinon from 'sinon';
 import { graphqlRequest } from '../db/connection';
 import {
   brandFactory,
+  customerFactory,
   engageMessageFactory,
   segmentFactory,
   tagsFactory,
@@ -205,6 +206,7 @@ describe('engageQueries', () => {
   });
 
   test('Enage email delivery report list', async () => {
+    const customer = await customerFactory();
     const dataSourceMock = sinon
       .stub(dataSources.EngagesAPI, 'engageReportsList')
       .callsFake(() => {
@@ -213,9 +215,13 @@ describe('engageQueries', () => {
             {
               _id: '123',
               status: 'pending'
+            },
+            {
+              _id: '234',
+              customerId: customer._id
             }
           ],
-          totalCount: 1
+          totalCount: 2
         });
       });
 
@@ -243,7 +249,7 @@ describe('engageQueries', () => {
       { dataSources }
     );
 
-    expect(response.list.length).toBe(1);
+    expect(response.list.length).toBe(2);
 
     dataSourceMock.restore();
   });
@@ -459,4 +465,24 @@ describe('engageQueries', () => {
       expect(e[0].message).toBe('Engages api is not running');
     }
   });
+
+  test('Test getAverageStats()', async () => {
+    const qry = `
+      query engageEmailPercentages {
+        engageEmailPercentages {
+          avgBouncePercent
+        }
+      }
+    `;
+
+    const mock = sinon
+      .stub(dataSources.EngagesAPI, 'getAverageStats')
+      .callsFake(() => {
+        return Promise.resolve({ data: { avgBouncePercent: 0 } });
+      });
+
+    await graphqlRequest(qry, 'engageEmailPercentages', {}, { dataSources });
+
+    mock.restore();
+  });
 });

api/src/data/constants.ts

@@ -292,3 +292,25 @@ export const AUTO_BOT_MESSAGES = {
 export const BOT_MESSAGE_TYPES = {
   SAY_SOMETHING: 'say_something'
 };
+
+export const AWS_EMAIL_STATUSES = {
+  SEND: 'send',
+  DELIVERY: 'delivery',
+  OPEN: 'open',
+  CLICK: 'click',
+  COMPLAINT: 'complaint',
+  BOUNCE: 'bounce',
+  RENDERING_FAILURE: 'renderingfailure',
+  REJECT: 'reject'
+};
+
+export const EMAIL_VALIDATION_STATUSES = {
+  VALID: 'valid',
+  INVALID: 'invalid',
+  ACCEPT_ALL_UNVERIFIABLE: 'accept_all_unverifiable',
+  UNVERIFIABLE: 'unverifiable',
+  UNKNOWN: 'unknown',
+  DISPOSABLE: 'disposable',
+  CATCH_ALL: 'catchall',
+  BAD_SYNTAX: 'badsyntax'
+};

api/src/data/dataSources/engages.ts

@@ -90,4 +90,17 @@ export default class EngagesAPI extends RESTDataSource {
       return {};
     }
   }
+
+  // fetches average email delivery stat percentages
+  public async getAverageStats() {
+    try {
+      const response = await this.get(`/deliveryReports/avgStatPercentages`);
+
+      return response;
+    } catch (e) {
+      debugBase(e);
+
+      return { error: e.message };
+    }
+  }
 } // end class

api/src/data/modules/integrations/receiveMessage.ts

@@ -8,6 +8,7 @@ import {
 } from '../../../db/models';
 import { CONVERSATION_STATUSES } from '../../../db/models/definitions/constants';
 import { graphqlPubsub } from '../../../pubsub';
+import { AWS_EMAIL_STATUSES, EMAIL_VALIDATION_STATUSES } from '../../constants';
 import { getConfigs } from '../../utils';
 
 const sendError = message => ({
@@ -172,10 +173,22 @@ export const receiveEngagesNotification = async msg => {
   const { action, data } = msg;
 
   if (action === 'setDoNotDisturb') {
-    await Customers.updateOne(
-      { _id: data.customerId },
-      { $set: { doNotDisturb: 'Yes' } }
-    );
+    const { customerId, status, customerIds = [] } = data;
+    const update: any = { doNotDisturb: 'Yes' };
+
+    if (status === AWS_EMAIL_STATUSES.BOUNCE) {
+      update.emailValidationStatus = EMAIL_VALIDATION_STATUSES.INVALID;
+    }
+
+    if (customerId) {
+      await Customers.updateOne({ _id: customerId }, { $set: update });
+    }
+    if (customerIds.length > 0 && !status) {
+      await Customers.updateMany(
+        { _id: { $in: customerIds } },
+        { $set: update }
+      );
+    }
   }
 
   if (action === 'transactionEmail') {

api/src/data/resolvers/mutations/engageUtils.ts

@@ -70,8 +70,8 @@ export const generateCustomerSelector = async ({
   }
 
   return {
-    $or: [{ doNotDisturb: 'No' }, { doNotDisturb: { $exists: false } }],
-    ...customerQuery
+    ...customerQuery,
+    $or: [{ doNotDisturb: 'No' }, { doNotDisturb: { $exists: false } }]
   };
 };
 

api/src/data/resolvers/queries/engages.ts

@@ -1,4 +1,4 @@
-import { EngageMessages, Tags } from '../../../db/models';
+import { Customers, EngageMessages, Tags } from '../../../db/models';
 import { IUserDocument } from '../../../db/models/definitions/users';
 import { checkPermission, requireLogin } from '../../permissions/wrappers';
 import { IContext } from '../../types';
@@ -32,6 +32,13 @@ interface ICountsByTag {
   [index: string]: number;
 }
 
+interface IReportParams {
+  page?: number;
+  perPage?: number;
+  customerId?: string;
+  status?: string;
+}
+
 // basic count helper
 const count = async (selector: {}): Promise<number> => {
   const res = await EngageMessages.find(selector).countDocuments();
@@ -224,8 +231,32 @@ const engageQueries = {
     return dataSources.EngagesAPI.engagesConfigDetail();
   },
 
-  engageReportsList(_root, params, { dataSources }: IContext) {
-    return dataSources.EngagesAPI.engageReportsList(params);
+  async engageReportsList(
+    _root,
+    params: IReportParams,
+    { dataSources }: IContext
+  ) {
+    const {
+      list = [],
+      totalCount
+    } = await dataSources.EngagesAPI.engageReportsList(params);
+    const modifiedList: any[] = [];
+
+    for (const item of list) {
+      const modifiedItem = item;
+
+      if (item.customerId) {
+        const customer = await Customers.findOne({ _id: item.customerId });
+
+        if (customer) {
+          modifiedItem.customerName = Customers.getCustomerName(customer);
+        }
+      }
+
+      modifiedList.push(modifiedItem);
+    }
+
+    return { totalCount, list: modifiedList };
   },
 
   /**
@@ -246,12 +277,19 @@ const engageQueries = {
    */
   engageVerifiedEmails(_root, _args, { dataSources }: IContext) {
     return dataSources.EngagesAPI.engagesGetVerifiedEmails();
+  },
+
+  async engageEmailPercentages(_root, _args, { dataSources }: IContext) {
+    const response = await dataSources.EngagesAPI.getAverageStats();
+
+    return response.data;
   }
 };
 
 requireLogin(engageQueries, 'engageMessagesTotalCount');
 requireLogin(engageQueries, 'engageMessageCounts');
 requireLogin(engageQueries, 'engageMessageDetail');
+requireLogin(engageQueries, 'engageEmailPercentages');
 
 checkPermission(engageQueries, 'engageMessages', 'showEngagesMessages', []);
 

api/src/data/schema/engage.ts

@@ -56,14 +56,27 @@ export const types = `
     mailId: String,
     status: String,
     engage: EngageMessage,
-    createdAt: Date
+    createdAt: Date,
+    customerName: String
   }
 
   type EngageDeliveryReport {
     list: [DeliveryReport]
     totalCount: Int
   }
 
+  type AvgEmailStats {
+    avgBouncePercent: Float,
+    avgClickPercent: Float,
+    avgComplaintPercent: Float,
+    avgDeliveryPercent: Float,
+    avgOpenPercent: Float,
+    avgRejectPercent: Float,
+    avgRenderingFailurePercent: Float,
+    avgSendPercent: Float,
+    total: Float
+  }
+
   input EngageScheduleDateInput {
     type: String,
     month: String,
@@ -114,7 +127,8 @@ export const queries = `
   engageMessageCounts(name: String!, kind: String, status: String): JSON
   engagesConfigDetail: JSON
   engageVerifiedEmails: [String]
-  engageReportsList(page: Int, perPage: Int): EngageDeliveryReport 
+  engageReportsList(page: Int, perPage: Int, customerId: String, status: String): EngageDeliveryReport 
+  engageEmailPercentages: AvgEmailStats
 `;
 
 const commonParams = `

engages-email-sender/package.json

@@ -12,6 +12,7 @@
     "@types/cors": "^2.8.4",
     "@types/dotenv": "^4.0.3",
     "@types/express": "^4.16.0",
+    "@types/faker": "^5.1.5",
     "@types/jest": "^24.0.23",
     "@types/mongodb": "^3.1.2",
     "@types/mongoose": "^5.2.1",
@@ -44,4 +45,4 @@
     "ts-node-dev": "^1.0.0-pre.32",
     "typescript": "^3.7.2"
   }
-}
+}

engages-email-sender/src/__tests__/deliveryReportsDb.test.ts

@@ -1,4 +1,5 @@
 import { DeliveryReports, Stats } from '../models';
+import { prepareAvgStats } from '../utils';
 import { statsFactory } from './factories';
 import './setup';
 
@@ -43,3 +44,21 @@ test('DeliveryReports: updateOrCreateReport', async done => {
 
   done();
 });
+
+test('Stats: Test average bounce & complaint percent', async done => {
+  const s1 = await statsFactory({});
+  const s2 = await statsFactory({});
+  const averageStat = await prepareAvgStats();
+
+  const bounce1 = (s1.bounce * 100) / s1.total;
+  const bounce2 = (s2.bounce * 100) / s2.total;
+  const complaint1 = (s1.complaint * 100) / s1.total;
+  const complaint2 = (s2.complaint * 100) / s2.total;
+
+  expect(averageStat[0].avgBouncePercent).toBe((bounce1 + bounce2) / 2);
+  expect(averageStat[0].avgComplaintPercent).toBe(
+    (complaint1 + complaint2) / 2
+  );
+
+  done();
+});

engages-email-sender/src/__tests__/factories.ts

@@ -1,5 +1,6 @@
 import * as faker from 'faker';
-import { Configs, Stats } from '../models';
+import { SES_DELIVERY_STATUSES } from '../constants';
+import { Configs, DeliveryReports, Stats } from '../models';
 
 /**
  * Returns random element of an array
@@ -19,7 +20,7 @@ export const configFactory = params => {
 
 export const statsFactory = params => {
   const statsObj = new Stats({
-    engageMessageId: params.engageMessageId || faker.random.id(),
+    engageMessageId: params.engageMessageId || faker.random.uuid(),
     open: params.open || faker.random.number(),
     click: params.click || faker.random.number(),
     complaint: params.complaint || faker.random.number(),
@@ -33,3 +34,34 @@ export const statsFactory = params => {
 
   return statsObj.save();
 };
+
+export const generateCustomerDoc = (params: any = {}) => ({
+  _id: faker.random.uuid(),
+  primaryEmail: params.email || faker.internet.email(),
+  emailValidationStatus: 'valid',
+  primaryPhone: faker.phone.phoneNumber(),
+  phoneValidationStatus: 'valid',
+  replacers: [{ key: 'key', value: 'value' }]
+});
+
+export const reportFactory = (params: any) => {
+  const report = new DeliveryReports({
+    customerId: params.customerId || faker.random.uuid(),
+    mailId: params.mailId || faker.random.uuid(),
+    status: params.status || randomElementOfArray(SES_DELIVERY_STATUSES.ALL),
+    engageMessageId: params.engageMesageId || faker.random.uuid(),
+    email: params.email || faker.internet.email()
+  });
+
+  return report.save();
+};
+
+export const generateCustomerDocList = (count: number) => {
+  const list: any[] = [];
+
+  for (let i = 0; i < count; i++) {
+    list.push(generateCustomerDoc());
+  }
+
+  return list;
+};