add timeout and abortsignal support for queries

Author: Razzmatazzz

src/tarkov-data-manager/jobs/update-item-cache.mjs

@@ -62,12 +62,14 @@ class UpdateItemCacheJob extends DataJob {
 
         const priceFields = [
             'lastLowPrice',
+            'lastLowAvgPrice',
             'avg24hPrice',
             'low24hPrice',
             'high24hPrice',
             'changeLast48h',
             'changeLast48hPercent',
             'lastOfferCount',
+            'updated',
         ];
         this.logger.log('Processing items...');
         for (const [key, value] of this.itemMap.entries()) {

src/tarkov-data-manager/modules/data-job.mjs

@@ -371,14 +371,26 @@ class DataJob {
         return this.translationHelper.hasTranslation(key, langCode);
     }
 
-    query = (sql, params) => {
-        const queryPromise = query(sql, params);
+    query = (sql, values, options) => {
+        if (typeof values === 'object' && !Array.isArray(values)) {
+            options = values;
+            values = undefined;
+        }
+        const queryPromise = query(sql, values, {signal: this.abortController.signal, ...options});
         this.queries.push(queryPromise);
         return queryPromise;
     }
 
-    batchQuery = (sql, params, batchCallback) => {
-        const queryPromise = batchQuery(sql, params, batchCallback);
+    batchQuery = (sql, values, batchCallback, options) => {
+        if (values && !Array.isArray(values)) {
+            batchCallback = values;
+            values = [];
+        }
+        if (!options && typeof batchCallback !== 'function') {
+            options = batchCallback;
+            batchCallback = undefined;
+        }
+        const queryPromise = batchQuery(sql, values, batchCallback, {signal: this.abortController.signal, ...options});
         this.queries.push(queryPromise);
         return queryPromise;
     }

src/tarkov-data-manager/modules/db-connection.mjs

@@ -59,30 +59,58 @@ const waitForConnections = () => {
 const dbConnection = {
     connection: pool,
     pool,
-    query: (sql, params) => {
+    query: async (sql, values, options = {}) => {
+        if (typeof values === 'object' && !Array.isArray(values)) {
+            options = values;
+            values = undefined;
+        }
+        let abortListener;
         return new Promise((resolve, reject) => {
+            abortListener = () => {
+                reject(new Error('Query aborted'));
+            };
+            options?.signal?.addEventListener('abort', abortListener, { once: true });
             try {
-                pool.query(sql, params, (error, results) => {
+                pool.query({sql, values, timeout: options?.timeout}, (error, results) => {
                     if (error) {
-                        reject(error);
-                        return;
+                        return reject(error);
                     }
-        
                     resolve(results);
                 });
             } catch (error) {
                 reject(error);
             }
+        }).finally(() => {
+            options.signal?.removeEventListener('abort', abortListener);
         });
     },
-    batchQuery: async (sql, params = [], batchCallback) => {
+    batchQuery: async (sql, values = [], batchCallback, options = {}) => {
         const batchSize = dbConnection.maxQueryRows;
         let offset = 0;
         const results = [];
+        const queryStart = new Date();
+        if (values && !Array.isArray(values)) {
+            batchCallback = values;
+            values = [];
+        }
+        if (!options && typeof batchCallback !== 'function') {
+            options = batchCallback;
+            batchCallback = undefined;
+        }
+        let timeout = options.timeout;
         while (true) {
-            const batchParams = [...params, offset, batchSize]
-            const batchResults = await dbConnection.query(`${sql} LIMIT ?, ?
-            `, batchParams);
+            const batchValues = [...values, offset, batchSize];
+            let batchTimeout;
+            if (timeout) {
+                batchTimeout = timeout - (new Date() - queryStart);
+                if (batchTimeout <= 0) {
+                    return Promise.reject(new Error('Query inactivity timeout'));
+                }
+            }
+            if (options.signal?.aborted) {
+                return Promise.reject(new Error('Query aborted'));
+            }
+            const batchResults = await dbConnection.query(`${sql} LIMIT ?, ?`, batchValues, {timeout: batchTimeout, signal: options.signal});
             batchResults.forEach(r => results.push(r));
             if (batchCallback) {
                 batchCallback(batchResults, offset);

src/tarkov-data-manager/modules/remote-data.mjs

@@ -130,11 +130,11 @@ const methods = {
                 logger.timeEnd('item-24h-price-query');
             });
 
-            logger.time('item-last-low-price-query');
-            const lastLowPricePromise = query(`
+            logger.time('item-last-price-query');
+            const lastPricePromise = query(`
                 SELECT
                     a.item_id,
-                    MIN(a.price) AS price,
+                    a.price,
                     timestamp,
                     a.game_mode
                 FROM
@@ -151,10 +151,8 @@ const methods = {
                 ) b
                 ON
                     a.item_id = b.item_id AND a.timestamp = b.max_timestamp AND a.game_mode = b.game_mode
-                GROUP BY
-                    a.item_id, a.timestamp, a.game_mode;
             `).finally(() => {
-                logger.timeEnd('item-last-low-price-query');
+                logger.timeEnd('item-last-price-query');
             });
 
             logger.time('price-yesterday-query');
@@ -178,12 +176,12 @@ const methods = {
             const [
                 items,
                 price24hResults,
-                lastLowPriceResults,
+                lastPriceResults,
                 avgPriceYesterday,
             ] = await Promise.all([
                 itemsPromise,
                 price24hPromise,
-                lastLowPricePromise,
+                lastPricePromise,
                 avgPriceYesterdayPromise,
             ]);
 
@@ -192,6 +190,20 @@ const methods = {
                 1: {},
             };
 
+            const itemLastPrices = lastPriceResults.reduce((all, current) => {
+                if (!all[current.game_mode]) {
+                    all[current.game_mode] = {};
+                }
+                if (!all[current.game_mode][current.item_id]) {
+                    all[current.game_mode][current.item_id] = {
+                        timestamp: current.timestamp,
+                        prices: [],
+                    };
+                }
+                all[current.game_mode][current.item_id].prices.push(current.price);
+                return all;
+            }, {});
+
             price24hResults.forEach((resultRow) => {
                 if (!item24hPrices[resultRow.game_mode][resultRow.item_id]) {
                     item24hPrices[resultRow.game_mode][resultRow.item_id] = [];
@@ -208,12 +220,16 @@ const methods = {
                 for (const gameMode of gameModes) {
                     const fieldPrefix = gameMode.name === 'regular' ? '' : `${gameMode.name}_`;
 
-                    const lastLowData = lastLowPriceResults.find(row => row.item_id === itemId && row.game_mode === gameMode.value);
-                    if (lastLowData) {
-                        item[`${fieldPrefix}lastLowPrice`] = lastLowData.price;
-                        if (gameMode.name === 'regular') {
-                            item.updated = lastLowData.timestamp;
-                        }
+                    //const lastLowData = lastLowPriceResults.find(row => row.item_id === itemId && row.game_mode === gameMode.value);
+                    const lastData = itemLastPrices[gameMode.value]?.[itemId];
+                    if (lastData) {
+                        item[`${fieldPrefix}lastLowPrice`] = lastData.prices[0];
+                        const lastLowAvgPrices = lastData.prices.slice(0, 3);
+                        item[`${fieldPrefix}lastLowAvgPrice`] = Math.round(lastLowAvgPrices.reduce((total, price) => {
+                            return total + price;
+                        }, 0) / lastLowAvgPrices.length);
+                        const updatedField = `${fieldPrefix}updated`;
+                        item[updatedField] = lastData.timestamp;
                     }
 
                     item24hPrices[gameMode.value][itemId]?.sort();

src/tarkov-data-manager/modules/scanner-api.mjs

@@ -279,7 +279,7 @@ const scannerApi = {
                 const oldScanners = users[user.username]?.scanners || [];
                 user.scanners = oldScanners;
                 users[user.username] = user;
-                scannerQueries.push(query('SELECT * from scanner WHERE scanner_user_id = ?', user.id).then(scanners => {
+                scannerQueries.push(query('SELECT * from scanner WHERE scanner_user_id = ?', [user.id]).then(scanners => {
                     users[user.username].scanners = scanners;
                 }));
             }

src/tarkov-data-manager/translations/en.json

@@ -113,6 +113,9 @@
     "quest_zone_keeper6_kiba_kill": "Around Kiba Arms store",
     "quest_zone_last_flare": "Armored Convoy",
     "quest_zone_prod_flare": "Pinewood Hotel Courtyard",
+    "sectantOni": "Oni",
+    "sectantPredvestnik": "Harbinger",
+    "sectantPrizrak": "Ghost",
     "SOPMOD II Flash": "SOPMOD II Flash",
     "ShturmanStash": "Shturman's Stash",
     "SPR": "SPR",
@@ -201,5 +204,7 @@
     "ZoneStartVillage": "Village",
     "ZoneSubCommand": "Command Bunker",
     "ZoneSubStorage": "Storage Bunker",
+    "ZoneSW00": "Zmejskij Alley Block",
+    "ZoneSW01": "Zmejskij Alley Block",
     "ZoneWoodCutter": "Lumber Mill"
 }