From f8e2e5c61ed42e180473b883f88327ea8e1c8803 Mon Sep 17 00:00:00 2001 From: Ludek Novy <13610612+ludeknovy@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:18:14 +0200 Subject: [PATCH] v5 (#320) --- src/db/Dockerfile | 4 +- .../item/shared/item-data-processing.ts | 19 +++-- src/server/queries/items.ts | 10 ++- src/server/utils/lttb.ts | 81 ------------------- 4 files changed, 17 insertions(+), 97 deletions(-) delete mode 100644 src/server/utils/lttb.ts diff --git a/src/db/Dockerfile b/src/db/Dockerfile index 0e9278b9..cfdeda11 100644 --- a/src/db/Dockerfile +++ b/src/db/Dockerfile @@ -1,3 +1,3 @@ -FROM timescale/timescaledb:2.4.1-pg13 +FROM timescale/timescaledb-ha:pg16.4-ts2.16.1-all ENV POSTGRES_DB jtl_report -COPY schema.sql /docker-entrypoint-initdb.d/ \ No newline at end of file +COPY schema.sql /docker-entrypoint-initdb.d/ diff --git a/src/server/controllers/item/shared/item-data-processing.ts b/src/server/controllers/item/shared/item-data-processing.ts index ccecccd6..c00412e2 100644 --- a/src/server/controllers/item/shared/item-data-processing.ts +++ b/src/server/controllers/item/shared/item-data-processing.ts @@ -24,11 +24,10 @@ import { updateItemApdexSettings, chartOverviewStatusCodesQuery, responseTimePerLabelHistogram, - findRawData, getBaselineItemWithStats, findGroupedErrors, findTop5ErrorsByLabel, - threadsPerThreadGroup, + threadsPerThreadGroup, getDownsampledRawData, } from "../../../queries/items" import { ReportStatus } from "../../../queries/items.model" import { getScenarioSettings } from "../../../queries/scenario" @@ -36,8 +35,6 @@ import { sendDegradationNotifications, sendReportNotifications } from "../../../ import { scenarioThresholdsCalc } from "../utils/scenario-thresholds-calc" import { extraIntervalMilliseconds } from "./extra-intervals-mapping" import { AnalyticsEvent } from "../../../utils/analytics/anyltics-event" -import { downsampleData } from "../../../utils/lttb" -import moment = require("moment"); import { DataProcessingException } from "../../../errors/data-processing-exceptions" export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) => { @@ -46,6 +43,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) let distributedThreads = null let sutMetrics = [] let apdex = [] + let rawDataArray = null try { const aggOverview = await db.one(aggOverviewQuery(itemId)) @@ -55,13 +53,12 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) const responseFailures = await db.manyOrNone(responseMessageFailures(itemId)) const scenarioSettings = await db.one(getScenarioSettings(projectName, scenarioName)) - let rawData = await db.manyOrNone(findRawData(itemId)) - let rawDataArray = rawData?.map(row => [moment(row.timestamp).valueOf(), row.elapsed]) - const rawDataDownSampled = downsampleData(rawDataArray, MAX_SCATTER_CHART_POINTS) + let rawDownsampledData = await db.manyOrNone(getDownsampledRawData(itemId, MAX_SCATTER_CHART_POINTS)) + rawDataArray = rawDownsampledData?.map(row => [row.timestamp, row.value]) + rawDownsampledData = null + const groupedErrors = await db.manyOrNone(findGroupedErrors(itemId)) const top5ErrorsByLabel = await db.manyOrNone(findTop5ErrorsByLabel(itemId)) - rawData = null - rawDataArray = null if (aggOverview.number_of_sut_hostnames > 1) { sutMetrics = await db.many(sutOverviewQuery(itemId)) @@ -153,7 +150,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) await t.none(saveItemStats(itemId, JSON.stringify(labelStats), overview, JSON.stringify(sutOverview), JSON.stringify(errors))) await t.none(savePlotData(itemId, JSON.stringify(chartData), JSON.stringify(extraChartData), - JSON.stringify(responseTimeHistogram), JSON.stringify(rawDataDownSampled))) + JSON.stringify(responseTimeHistogram), JSON.stringify(rawDataArray))) await t.none(updateItem(itemId, ReportStatus.Ready, overview.startDate)) }) @@ -170,5 +167,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) } catch(error) { throw new DataProcessingException( `Error while processing dataId: ${itemId} for item: ${itemId}, error: ${error}`) + } finally { + rawDataArray = [] } } diff --git a/src/server/queries/items.ts b/src/server/queries/items.ts index b0e1d014..8f8a73a6 100644 --- a/src/server/queries/items.ts +++ b/src/server/queries/items.ts @@ -59,11 +59,13 @@ export const findItemStats = (testItem) => { } } -export const findRawData = (itemId) => { +export const getDownsampledRawData = (itemId: string, threshold: number) => { return { - text: `SELECT timestamp, elapsed - FROM jtl.samples WHERE item_id = $1`, - values: [itemId], + text: `SELECT (EXTRACT(epoch FROM time) * 1000)::bigint as timestamp, value + FROM unnest(( + SELECT lttb(timestamp, elapsed, $2) + FROM jtl.samples WHERE item_id = $1));`, + values: [itemId, threshold], } } diff --git a/src/server/utils/lttb.ts b/src/server/utils/lttb.ts deleted file mode 100644 index baa7d4ee..00000000 --- a/src/server/utils/lttb.ts +++ /dev/null @@ -1,81 +0,0 @@ -// https://github.com/pingec/downsample-lttb -export const downsampleData = (series, threshold) => { - return largestTriangleThreeBuckets(series, threshold) -} - -const floor = Math.floor, - abs = Math.abs - -function largestTriangleThreeBuckets(data, threshold) { - - const dataLength = data.length - if (threshold >= dataLength || threshold === 0) { - return data // Nothing to do - } - - const sampled = [] - let sampledIndex = 0 - - // Bucket size. Leave room for start and end data points - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - const every = (dataLength - 2) / (threshold - 2) - - let a = 0, // Initially a is the first point in the triangle - maxAreaPoint, - maxArea, - area, - nextA - - sampled[sampledIndex++] = data[a] // Always add the first point - - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - for (let i = 0; i < threshold - 2; i++) { - - // Calculate point average for next bucket (containing c) - let avgX = 0, - avgY = 0, - avgRangeStart = floor( ( i + 1 ) * every ) + 1, - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - avgRangeEnd = floor( ( i + 2 ) * every ) + 1 - avgRangeEnd = avgRangeEnd < dataLength ? avgRangeEnd : dataLength - - const avgRangeLength = avgRangeEnd - avgRangeStart - - for ( ; avgRangeStart < avgRangeEnd; avgRangeStart++ ) { - avgX += data[avgRangeStart][0] * 1 // * 1 enforces Number (value may be Date) - avgY += data[avgRangeStart][1] * 1 - } - avgX /= avgRangeLength - avgY /= avgRangeLength - - // Get the range for this bucket - let rangeOffs = floor( (i + 0) * every ) + 1 - const rangeTo = floor( (i + 1) * every ) + 1 - - // Point a - const pointAX = data[a][0] * 1, // enforce Number (value may be Date) - pointAY = data[a][1] * 1 - - maxArea = area = -1 - - for ( ; rangeOffs < rangeTo; rangeOffs++ ) { - // Calculate triangle area over three buckets - area = abs( ( pointAX - avgX ) * ( data[rangeOffs][1] - pointAY ) - - ( pointAX - data[rangeOffs][0] ) * ( avgY - pointAY ) - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - ) * 0.5 - if ( area > maxArea ) { - maxArea = area - maxAreaPoint = data[rangeOffs] - nextA = rangeOffs // Next a is this b - } - } - - sampled[sampledIndex++] = maxAreaPoint // Pick this point from the bucket - a = nextA // This a is the next a (chosen b) - } - - sampled[sampledIndex++] = data[dataLength - 1] // Always add last - - return sampled -}