From 7cc13359134224cef6963a97b68e19a434d505be Mon Sep 17 00:00:00 2001 From: cristianpb <18708179+cristianpb@users.noreply.github.com> Date: Mon, 2 Jun 2025 22:52:38 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20Use=20expect=20and=20ensure=20i?= =?UTF-8?q?n=20artillery=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- .../performance/scenarios/test-backend-v1.yml | 29 ++++++++++++++++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bce2f455..cfe618b8 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,7 @@ export PERF=${BACKEND}/tests/performance export PERF_SCENARIO_V1=${PERF}/scenarios/test-backend-v1.yml export PERF_REPORTS=${PERF}/reports/ export PERF_NAMES=${BACKEND}/tests/clients_test.csv +export PERF_UTILS=${PERF}/scenarios/utils -include ${APP_PATH}/${GIT_TOOLS}/artifacts.SCW export SCW_REGION?=fr-par @@ -334,7 +335,6 @@ backend/tests/clients_test.csv: # test artillery test-perf-v1: - sed -i -E "s/;/,/g" backend/tests/clients_test.csv make -C ${APP_PATH}/${GIT_TOOLS} test-api-generic PERF_SCENARIO=${PERF_SCENARIO_V1} PERF_TEST_ENV=api-perf PERF_REPORTS=${PERF_REPORTS} DC_NETWORK=${DC_NETWORK} PERF_NAMES=${PERF_NAMES}; backend-perf-clinic: diff --git a/backend/tests/performance/scenarios/test-backend-v1.yml b/backend/tests/performance/scenarios/test-backend-v1.yml index 3344f8ab..95f8076e 100644 --- a/backend/tests/performance/scenarios/test-backend-v1.yml +++ b/backend/tests/performance/scenarios/test-backend-v1.yml @@ -1,4 +1,12 @@ config: + plugins: + expect: + reportFailuresAsErrors: true + outputFormat: prettyError + ensure: + maxErrorRate: 1 + max: 500 + p95: 200 environments: api-perf: target: "http://backend:8080" @@ -18,24 +26,37 @@ config: payload: - path: names.csv fields: - - firstName + - lastName - id - country - commune - birthDate - - lastName + - firstName order: random - # the following doesn't work skipHeader: true - #delimiter: ";" + delimiter: ";" scenarios: - name: backend-v1 flow: - get: url: "/deces/api/v1/search?firstName={{ firstName }}&lastName={{ lastName }}&birthDate={{ birthDate }}&fuzzy=false" + expect: + - statusCode: 200 + - contentType: json + capture: + - json: '$' + as: resultGet + #- log: "Res GET: {{ resultGet }}" - post: url: /deces/api/v1/search json: firstName: "{{ firstName }}" lastName: "{{ lastName }}" birthDate: "{{ birthDate }}" + capture: + - json: '$' + as: resultPost + expect: + - statusCode: 200 + - contentType: json + #- log: "Res POST: {{ resultPost }}" From 8d63a006d42b66861f2ea289227de2f22e6181f6 Mon Sep 17 00:00:00 2001 From: cristianpb <18708179+cristianpb@users.noreply.github.com> Date: Mon, 2 Jun 2025 22:53:35 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=8F=B7=20Update=20aggregation=20typin?= =?UTF-8?q?gs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/buildRequest.ts | 4 +-- .../src/controllers/aggregation.controller.ts | 32 +++++++++++-------- backend/src/models/result.ts | 12 ++++++- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/backend/src/buildRequest.ts b/backend/src/buildRequest.ts index 3bbba11b..2ff028df 100644 --- a/backend/src/buildRequest.ts +++ b/backend/src/buildRequest.ts @@ -460,14 +460,14 @@ const buildAggregation = (aggs: string[], aggsSize: number, afterKey: number): a } return aggregation }) - aggregationRequest.myBuckets = { + aggregationRequest.bucketResults = { composite: { size: 1000, sources: aggregationArray } } if (afterKey !== undefined) { - aggregationRequest.myBuckets.composite.after = afterKey + aggregationRequest.bucketResults.composite.after = afterKey } else { aggs.map((agg: string) => { aggregationRequest[`${agg}_count`] = { diff --git a/backend/src/controllers/aggregation.controller.ts b/backend/src/controllers/aggregation.controller.ts index 64c78981..76eeeae3 100644 --- a/backend/src/controllers/aggregation.controller.ts +++ b/backend/src/controllers/aggregation.controller.ts @@ -76,7 +76,7 @@ export class AggregationController extends Controller { this.setStatus(400); return { msg: "error - simple and complex request at the same time" }; } - await this.streamAggs((request).res, requestInput, accept) + await this.streamAggs(request.res, requestInput, accept) } /** @@ -107,10 +107,10 @@ export class AggregationController extends Controller { this.setStatus(400); return { msg: requestInput.errors }; } - await this.streamAggs((request).res, requestInput, accept) + await this.streamAggs(request.res, requestInput, accept) } - private async streamAggs(response: any, requestInput: any, accept: string) { + private async streamAggs(response: express.Response, requestInput: RequestInput, accept: string) { let requestBuild = buildRequest(requestInput); const transformedAggs = requestInput.aggs.mask.transform(requestInput.aggs.value) let result = await runRequest(requestBuild, null); @@ -119,13 +119,17 @@ export class AggregationController extends Controller { let buckets const cardinality: any = {} let { took: delay } = result - if (result.aggregations.myBuckets) { - afterKey = result.aggregations.myBuckets.after_key + if (result.error) { + this.setStatus(result.status); + return { msg: result.error }; + } + if (result.aggregations.bucketResults) { + afterKey = result.aggregations.bucketResults.after_key transformedAggs.forEach((agg: string) => { cardinality[agg] = result.aggregations[`${agg}_count`].value response.setHeader(`total-results-${agg}`, result.aggregations[`${agg}_count`].value); }); - buckets = result.aggregations.myBuckets.buckets + buckets = result.aggregations.bucketResults.buckets } else { transformedAggs.forEach((agg: string) => { cardinality[agg] = result.aggregations[agg].buckets.length @@ -138,7 +142,7 @@ export class AggregationController extends Controller { if (buckets.length > 0) { buckets.forEach((bucketItem: any, ind: number) => { const aggKeys: any = {} - if (result.aggregations.myBuckets) { + if (result.aggregations.bucketResults) { Object.entries(bucketItem.key).forEach(([key, value]) => { aggKeys[key] = value }) @@ -155,13 +159,13 @@ export class AggregationController extends Controller { response.write(Object.values(aggKeys).join(",") + '\n') }) } - while (result.aggregations.myBuckets && result.aggregations.myBuckets.buckets.length > 0 ) { + while (result.aggregations.bucketResults && result.aggregations.bucketResults.buckets.length > 0 ) { requestInput.afterKey = afterKey requestBuild = buildRequest(requestInput); result = await runRequest(requestBuild, null); - afterKey = result.aggregations.myBuckets.after_key + afterKey = result.aggregations.bucketResults.after_key delay += result.took - const { buckets: afterBucket } = result.aggregations.myBuckets + const { buckets: afterBucket } = result.aggregations.bucketResults if (afterBucket.length > 0 ) { afterBucket.forEach((bucketItem: any) => { const aggKeys: any = {} @@ -202,18 +206,18 @@ export class AggregationController extends Controller { response.write(JSON.stringify(firstItem[0])) buckets.forEach((bucketItem: any) => response.write("," + JSON.stringify(bucketItem))) } - while (result.aggregations.myBuckets && result.aggregations.myBuckets.buckets.length > 0 ) { + while (result.aggregations.bucketResults && result.aggregations.bucketResults.buckets.length > 0 ) { requestInput.afterKey = afterKey requestBuild = buildRequest(requestInput); result = await runRequest(requestBuild, null); - afterKey = result.aggregations.myBuckets.after_key + afterKey = result.aggregations.bucketResults.after_key delay += result.took - const { buckets: afterBucket } = result.aggregations.myBuckets + const { buckets: afterBucket } = result.aggregations.bucketResults if (afterBucket.length > 0 ) { afterBucket.forEach((bucketItem: any) => response.write("," + JSON.stringify(bucketItem))) } } - response.write(`],"delay": ${delay as string}}}`) + response.write(`],"delay": ${String(delay)}}}`) response.end(); } } diff --git a/backend/src/models/result.ts b/backend/src/models/result.ts index d5a00131..4b6ec372 100644 --- a/backend/src/models/result.ts +++ b/backend/src/models/result.ts @@ -148,10 +148,20 @@ export interface ResultRawES { hits: ResultRawHit[] }; aggregations?: { + [key: string]: any; 'doc_count_error_upper_bound': number; 'sum_other_doc_count': number; - buckets: any[]; + bucketResults: { + buckets: { + key: string; + doc_count: number; + }[]; + after_key: number; + } } + status?: number; + statusText?: string; + error?: boolean; } export interface ResultRawHit { From 06f2b5388fc563ad846aef7eb93b5e9c3dd91609 Mon Sep 17 00:00:00 2001 From: cristianpb <18708179+cristianpb@users.noreply.github.com> Date: Mon, 2 Jun 2025 22:56:51 +0200 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9C=A8=20Use=20typings=20in=20runRequest?= =?UTF-8?q?=20elasticsearch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/runRequest.ts | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/backend/src/runRequest.ts b/backend/src/runRequest.ts index 12631248..d71028ab 100644 --- a/backend/src/runRequest.ts +++ b/backend/src/runRequest.ts @@ -1,6 +1,7 @@ import { getClient } from './elasticsearch'; import loggerStream from './logger'; import { BodyResponse, ScrolledResponse } from './models/body'; +import { ResultRawES } from './models/result'; const log = (json:any) => { loggerStream.write(JSON.stringify({ @@ -15,7 +16,7 @@ const isBodyResponse = (body: BodyResponse|ScrolledResponse): body is BodyRespon return 'query' in body; } -export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: string): Promise => { +export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: string): Promise => { const client = getClient(); let operation = 'unknown'; try { @@ -52,26 +53,26 @@ export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: st }); return { + took: 0, hits: { total: { - value: 1 + value: 0 }, - hits: [ - { - _id: 0, - _source: { - status: error.statusCode || 500, - statusText: error.message || 'Internal Server Error', - error: true - } - } - ] - } + max_score: 0, + hits: [] + }, + status: error.statusCode || 500, + statusText: error.message || 'Internal Server Error', + error: true }; } }; -export const runBulkRequest = async (bulkRequest: any): Promise => { +interface ResultBulkES { + responses: ResultRawES[]; +} + +export const runBulkRequest = async (bulkRequest: any): Promise => { const client = getClient(); try { const response = await client.msearch(bulkRequest); From c8235c73ccd8967aea7f4621a7cce997f4463015 Mon Sep 17 00:00:00 2001 From: cristianpb <18708179+cristianpb@users.noreply.github.com> Date: Wed, 11 Jun 2025 22:26:08 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=8F=B7=20Use=20estypes=20for=20bulk?= =?UTF-8?q?=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/src/runRequest.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/src/runRequest.ts b/backend/src/runRequest.ts index d71028ab..02da523e 100644 --- a/backend/src/runRequest.ts +++ b/backend/src/runRequest.ts @@ -1,7 +1,7 @@ +import { estypes } from '@elastic/elasticsearch'; import { getClient } from './elasticsearch'; import loggerStream from './logger'; import { BodyResponse, ScrolledResponse } from './models/body'; -import { ResultRawES } from './models/result'; const log = (json:any) => { loggerStream.write(JSON.stringify({ @@ -16,7 +16,7 @@ const isBodyResponse = (body: BodyResponse|ScrolledResponse): body is BodyRespon return 'query' in body; } -export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: string): Promise => { +export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: string): Promise => { const client = getClient(); let operation = 'unknown'; try { @@ -68,11 +68,7 @@ export const runRequest = async (body: BodyResponse|ScrolledResponse, scroll: st } }; -interface ResultBulkES { - responses: ResultRawES[]; -} - -export const runBulkRequest = async (bulkRequest: any): Promise => { +export const runBulkRequest = async (bulkRequest: any): Promise => { const client = getClient(); try { const response = await client.msearch(bulkRequest);