using db with timescale toolkit pre-installed, lttb module removed an…

…d replaced by db function to avoid high memory usage

migrations/1713374194350_toolkit.js

@@ -0,0 +1,3 @@
+exports.up = (pgm) => {
+    pgm.createExtension("timescaledb_toolkit")
+}

src/db/Dockerfile

@@ -1,3 +1,3 @@
-FROM timescale/timescaledb:2.4.1-pg13
+FROM novyl/jtl-reporter-db:latest
 ENV POSTGRES_DB jtl_report
-COPY schema.sql /docker-entrypoint-initdb.d/
+COPY schema.sql /docker-entrypoint-initdb.d/

src/server/controllers/item/shared/item-data-processing.ts

@@ -24,19 +24,17 @@ import {
     updateItemApdexSettings,
     chartOverviewStatusCodesQuery,
     responseTimePerLabelHistogram,
-    findRawData,
     getBaselineItemWithStats,
     findGroupedErrors,
     findTop5ErrorsByLabel,
-    threadsPerThreadGroup,
+    threadsPerThreadGroup, getDownsampledRawData,
 } from "../../../queries/items"
 import { ReportStatus } from "../../../queries/items.model"
 import { getScenarioSettings } from "../../../queries/scenario"
 import { sendDegradationNotifications, sendReportNotifications } from "../../../utils/notifications/send-notification"
 import { scenarioThresholdsCalc } from "../utils/scenario-thresholds-calc"
 import { extraIntervalMilliseconds } from "./extra-intervals-mapping"
 import { AnalyticsEvent } from "../../../utils/analytics/anyltics-event"
-import { downsampleData } from "../../../utils/lttb"
 import moment = require("moment");
 import { DataProcessingException } from "../../../errors/data-processing-exceptions"
 
@@ -46,6 +44,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId })
     let distributedThreads = null
     let sutMetrics = []
     let apdex = []
+    let rawDataArray = null
 
     try {
         const aggOverview = await db.one(aggOverviewQuery(itemId))
@@ -55,13 +54,12 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId })
         const responseFailures = await db.manyOrNone(responseMessageFailures(itemId))
         const scenarioSettings = await db.one(getScenarioSettings(projectName, scenarioName))
 
-        let rawData = await db.manyOrNone(findRawData(itemId))
-        let rawDataArray = rawData?.map(row => [moment(row.timestamp).valueOf(), row.elapsed])
-        const rawDataDownSampled = downsampleData(rawDataArray, MAX_SCATTER_CHART_POINTS)
+        let rawDownsampledData = await db.manyOrNone(getDownsampledRawData(itemId, MAX_SCATTER_CHART_POINTS))
+        rawDataArray = rawDownsampledData?.map(row => [row.timestamp, row.value])
+        rawDownsampledData = null
+
         const groupedErrors = await db.manyOrNone(findGroupedErrors(itemId))
         const top5ErrorsByLabel = await db.manyOrNone(findTop5ErrorsByLabel(itemId))
-        rawData = null
-        rawDataArray = null
 
         if (aggOverview.number_of_sut_hostnames > 1) {
             sutMetrics = await db.many(sutOverviewQuery(itemId))
@@ -153,7 +151,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId })
             await t.none(saveItemStats(itemId, JSON.stringify(labelStats),
                 overview, JSON.stringify(sutOverview), JSON.stringify(errors)))
             await t.none(savePlotData(itemId, JSON.stringify(chartData), JSON.stringify(extraChartData),
-                JSON.stringify(responseTimeHistogram), JSON.stringify(rawDataDownSampled)))
+                JSON.stringify(responseTimeHistogram), JSON.stringify(rawDataArray)))
             await t.none(updateItem(itemId, ReportStatus.Ready, overview.startDate))
         })
 
@@ -170,5 +168,7 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId })
     } catch(error) {
         throw new DataProcessingException(
             `Error while processing dataId: ${itemId} for item: ${itemId}, error: ${error}`)
+    } finally {
+        rawDataArray = []
     }
 }

src/server/queries/items.ts

@@ -59,11 +59,13 @@ export const findItemStats = (testItem) => {
     }
 }
 
-export const findRawData = (itemId) => {
+export const getDownsampledRawData = (itemId: string, threshold: number) => {
     return {
-        text: `SELECT timestamp, elapsed
-            FROM jtl.samples WHERE item_id = $1`,
-        values: [itemId],
+        text: `SELECT (EXTRACT(epoch FROM time) * 1000)::bigint as timestamp, value
+                FROM unnest((
+                    SELECT lttb(timestamp, elapsed, $2)
+                    FROM jtl.samples WHERE item_id = $1));`,
+        values: [itemId, threshold],
     }
 }