From d24a17bff66dc1a8f71ad584ef470e4d41db2a25 Mon Sep 17 00:00:00 2001 From: zhq745w Date: Tue, 19 Nov 2024 15:51:04 +0300 Subject: [PATCH 1/6] Add timestamp to log (#350) --- src/logger.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/logger.ts b/src/logger.ts index d30d28d1..b45f0899 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,6 +1,10 @@ import * as winston from "winston" export const logger = winston.createLogger({ + format: winston.format.combine( + winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), + winston.format.json() + ), levels: winston.config.syslog.levels, transports: [ new winston.transports.Console() ], }) From b3e27d25c980dad5eae1f3f6e280b9418f503763 Mon Sep 17 00:00:00 2001 From: Ludek Novy <13610612+ludeknovy@users.noreply.github.com> Date: Sun, 8 Dec 2024 12:44:49 +0100 Subject: [PATCH 2/6] fix: package.json & package-lock.json to reduce vulnerabilities (#352) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-PATHTOREGEXP-8482416 Co-authored-by: snyk-bot --- package-lock.json | 29 +++++++++++++++++------------ package.json | 2 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7442ad47..b38436f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "bree": "^9.2.4", "compression": "^1.7.4", "dotenv": "^16.4.5", - "express": "^4.21.0", + "express": "^4.21.2", "express-winston": "^4.2.0", "fast-csv": "^4.3.6", "helmet": "^6.2.0", @@ -4021,9 +4021,10 @@ "dev": true }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "license": "MIT", "engines": { "node": ">= 0.6" } @@ -4809,9 +4810,9 @@ } }, "node_modules/express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", "license": "MIT", "dependencies": { "accepts": "~1.3.8", @@ -4819,7 +4820,7 @@ "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -4833,7 +4834,7 @@ "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.10", + "path-to-regexp": "0.1.12", "proxy-addr": "~2.0.7", "qs": "6.13.0", "range-parser": "~1.2.1", @@ -4848,6 +4849,10 @@ }, "engines": { "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/express-winston": { @@ -9541,9 +9546,9 @@ "dev": true }, "node_modules/path-to-regexp": { - "version": "0.1.10", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", - "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "license": "MIT" }, "node_modules/path-type": { diff --git a/package.json b/package.json index d18cc21b..6efb10ad 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "bree": "^9.2.4", "compression": "^1.7.4", "dotenv": "^16.4.5", - "express": "^4.21.0", + "express": "^4.21.2", "express-winston": "^4.2.0", "fast-csv": "^4.3.6", "helmet": "^6.2.0", From e91426510817dfa8bf8dcfa99353aeac43bd6d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lud=C4=9Bk=20Nov=C3=BD?= <13610612+ludeknovy@users.noreply.github.com> Date: Tue, 10 Dec 2024 17:38:24 +0100 Subject: [PATCH 3/6] Change log level from info to debug in authorization middleware --- src/server/middleware/authorization-middleware.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server/middleware/authorization-middleware.ts b/src/server/middleware/authorization-middleware.ts index 802a7584..de459517 100644 --- a/src/server/middleware/authorization-middleware.ts +++ b/src/server/middleware/authorization-middleware.ts @@ -6,14 +6,14 @@ import { isUserAuthorizedForProject } from "../queries/user-project-access" export const authorizationMiddleware = (allowedRoles: AllowedRoles[]) => { return async (request, response, next) => { const user = request.user - logger.info(`User ${user.userId} with role ${user.role} accessing a resource with allowed roles: ${allowedRoles}`) + logger.debug(`User ${user.userId} with role ${user.role} accessing a resource with allowed roles: ${allowedRoles}`) // check project authorization const { projectName } = request.params if (projectName && user?.userId && user?.role !== AllowedRoles.Admin) { - logger.info(`User ${user.userId} with role ${user.role} accessing a resource within ${projectName} project`) + logger.debug(`User ${user.userId} with role ${user.role} accessing a resource within ${projectName} project`) const userAuthorizedForProject = await db.oneOrNone(isUserAuthorizedForProject(projectName, user.userId)) if (!userAuthorizedForProject && user.role) { - logger.info(`User ${user.userId} has no access to project ${projectName}`) + logger.debug(`User ${user.userId} has no access to project ${projectName}`) return next(boom.forbidden(`You dont have permission to access`)) } // user is authorized, we can proceed From 5cd23f30b5d3c00c4a428d901289290691f388cc Mon Sep 17 00:00:00 2001 From: Ludek Novy <13610612+ludeknovy@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:09:59 +0100 Subject: [PATCH 4/6] Add debug logs for data loading and processing steps (#353) --- .../item/shared/item-data-processing.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/server/controllers/item/shared/item-data-processing.ts b/src/server/controllers/item/shared/item-data-processing.ts index c00412e2..14be2f28 100644 --- a/src/server/controllers/item/shared/item-data-processing.ts +++ b/src/server/controllers/item/shared/item-data-processing.ts @@ -46,29 +46,40 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) let rawDataArray = null try { + logger.debug("Loading overview aggregation") const aggOverview = await db.one(aggOverviewQuery(itemId)) + logger.debug("Loading label aggregation") const aggLabel = await db.many(aggLabelQuery(itemId)) + logger.debug("Loading status code distribution") const statusCodeDistribution = await db.manyOrNone(responseCodeDistribution(itemId)) + logger.debug("Loading response time per label distribution") const responseTimePerLabelDistribution = await db.manyOrNone(responseTimePerLabelHistogram(itemId)) + logger.debug("Loading response failures") const responseFailures = await db.manyOrNone(responseMessageFailures(itemId)) + logger.debug("Loading scenario settings") const scenarioSettings = await db.one(getScenarioSettings(projectName, scenarioName)) - + logger.debug("Loading raw downsampled data") let rawDownsampledData = await db.manyOrNone(getDownsampledRawData(itemId, MAX_SCATTER_CHART_POINTS)) rawDataArray = rawDownsampledData?.map(row => [row.timestamp, row.value]) rawDownsampledData = null + logger.debug("Loading grouped errors") const groupedErrors = await db.manyOrNone(findGroupedErrors(itemId)) + logger.debug("Loading top 5 errors by label") const top5ErrorsByLabel = await db.manyOrNone(findTop5ErrorsByLabel(itemId)) if (aggOverview.number_of_sut_hostnames > 1) { + logger.debug("Loading SUT overview") sutMetrics = await db.many(sutOverviewQuery(itemId)) } if (scenarioSettings.apdexSettings.enabled) { const { satisfyingThreshold, toleratingThreshold } = scenarioSettings.apdexSettings + logger.debug("Calculating apdex") apdex = await db.many(calculateApdexValues(itemId, satisfyingThreshold, toleratingThreshold)) + logger.debug("Updating apdex settings") await db.none(updateItemApdexSettings(itemId, { satisfyingThreshold, toleratingThreshold, @@ -92,13 +103,18 @@ export const itemDataProcessing = async ({ projectName, scenarioName, itemId }) // distributed mode if (aggOverview?.number_of_hostnames > 1) { + logger.debug("Loading distributed threads") distributedThreads = await db.manyOrNone(distributedThreadsQuery(interval, itemId)) } + logger.debug("Loading label chart") const labelChart = await db.many(charLabelQuery(interval, itemId)) + logger.debug("Loading overview chart") const overviewChart = await db.many(chartOverviewQuery(interval, itemId)) + logger.debug("Loading status code chart") const statusCodeChart = await db.many(chartOverviewStatusCodesQuery(interval, itemId)) + logger.debug("Loading threads per group") const threadsPerGroup = await db.manyOrNone(threadsPerThreadGroup(interval, itemId)) if (parseInt(index, 10) === 0) { // default interval chartData = prepareChartDataForSaving( From d60d8b23a606d1b791fc1ffa9c4e92fdebbc93ba Mon Sep 17 00:00:00 2001 From: Ludek Novy <13610612+ludeknovy@users.noreply.github.com> Date: Wed, 11 Dec 2024 10:10:43 +0100 Subject: [PATCH 5/6] Add dynamic log level configuration with environment variable (#354) --- src/logger.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/logger.ts b/src/logger.ts index b45f0899..45026cb2 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,10 +1,32 @@ import * as winston from "winston" +const LOG_LEVELS = winston.config.syslog.levels +const DEFAULT_LOG_LEVEL = "debug" + +const getLogLevel = (): string => { + const logLevelEnvVar = process.env.LOG_LEVEL + if (logLevelEnvVar) { + const isAllowedLogLevel = Object.keys(LOG_LEVELS) + .find(level => logLevelEnvVar.toLowerCase() === level.toLowerCase()) + if (!isAllowedLogLevel) { + console.log("Unsupported log level: ", logLevelEnvVar) + } else { + return isAllowedLogLevel + } + } + return DEFAULT_LOG_LEVEL + + +} + export const logger = winston.createLogger({ - format: winston.format.combine( - winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), - winston.format.json() - ), - levels: winston.config.syslog.levels, - transports: [ new winston.transports.Console() ], + format: winston.format.combine( + winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), + winston.format.json() + ), + level: getLogLevel(), + levels: LOG_LEVELS, + transports: [new winston.transports.Console()], }) + + From 6a1fed6873ee4878d415601e4657452e35e77409 Mon Sep 17 00:00:00 2001 From: Ludek Novy <13610612+ludeknovy@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:42:54 +0100 Subject: [PATCH 6/6] fix: package.json & package-lock.json to reduce vulnerabilities (#351) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-AXIOS-6671926 Co-authored-by: snyk-bot --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index b38436f6..ab709070 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "ISC", "dependencies": { "@breejs/ts-worker": "^2.0.0", - "axios": "^1.7.5", + "axios": "^1.7.8", "bcrypt": "^5.1.1", "body-parser": "^1.20.3", "boom": "^7.2.0", @@ -3312,9 +3312,9 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "node_modules/axios": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.5.tgz", - "integrity": "sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==", + "version": "1.7.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz", + "integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==", "license": "MIT", "dependencies": { "follow-redirects": "^1.15.6", diff --git a/package.json b/package.json index 6efb10ad..1b790a95 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "license": "ISC", "dependencies": { "@breejs/ts-worker": "^2.0.0", - "axios": "^1.7.5", + "axios": "^1.7.8", "bcrypt": "^5.1.1", "body-parser": "^1.20.3", "boom": "^7.2.0",