feat(metrics): add metrics logging and configuration support (#1559)

* feat(metrics): add metrics logging and configuration support

* refactor folder structure

* implement accesslogs service methods

* minor fixes

* metrics service init part

* remove metrics modules and send logs to the graylog

* renamings

* remove unused package

* update example config

* add comments for conditional module loading and env variable reuse

* improved error message for when no/wrong format json file provided

Author: Junjiequan

.gitignore

@@ -5,6 +5,7 @@ functionalAccounts.json
 datasetTypes.json
 proposalTypes.json
 loggers.json
+metricsConfig.json
 
 # Configs
 .env

metricsConfig.example.json

@@ -0,0 +1,31 @@
+{
+  "include": [
+    {
+      "path": "*",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/fullquery",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/:id",
+      "method": 0,
+      "version": "3"
+    }
+  ],
+  "exclude": [
+    {
+      "path": "datasets/fullfacet",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/metadataKeys",
+      "method": 0,
+      "version": "3"
+    }
+  ]
+}

src/app.module.ts

@@ -3,7 +3,7 @@ import { MongooseModule } from "@nestjs/mongoose";
 import { DatasetsModule } from "./datasets/datasets.module";
 import { AuthModule } from "./auth/auth.module";
 import { UsersModule } from "./users/users.module";
-import { ConfigModule, ConfigService } from "@nestjs/config";
+import { ConditionalModule, ConfigModule, ConfigService } from "@nestjs/config";
 import { CaslModule } from "./casl/casl.module";
 import configuration from "./config/configuration";
 import { APP_GUARD, Reflector } from "@nestjs/core";
@@ -32,6 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter";
 import { AdminModule } from "./admin/admin.module";
 import { HealthModule } from "./health/health.module";
 import { LoggerModule } from "./loggers/logger.module";
+import { MetricsModule } from "./metrics/metrics.module";
 
 @Module({
   imports: [
@@ -42,6 +43,13 @@ import { LoggerModule } from "./loggers/logger.module";
     ConfigModule.forRoot({
       load: [configuration],
     }),
+    // NOTE: `ConditionalModule.registerWhen` directly uses `process.env` as it does not support
+    // dependency injection for `ConfigService`. This approach ensures compatibility while
+    // leveraging environment variables for conditional module loading.
+    ConditionalModule.registerWhen(
+      MetricsModule,
+      (env: NodeJS.ProcessEnv) => env.METRICS_ENABLED === "yes",
+    ),
     LoggerModule,
     DatablocksModule,
     DatasetsModule,

src/config/configuration.ts

@@ -48,6 +48,7 @@ const configuration = () => {
     loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json",
     datasetTypes: process.env.DATASET_TYPES_FILE || "datasetTypes.json",
     proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json",
+    metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json",
   };
   Object.keys(jsonConfigFileList).forEach((key) => {
     const filePath = jsonConfigFileList[key];
@@ -204,7 +205,13 @@ const configuration = () => {
       mongoDBCollection: process.env.MONGODB_COLLECTION,
       defaultIndex: process.env.ES_INDEX ?? "dataset",
     },
-
+    metrics: {
+      // Note: `process.env.METRICS_ENABLED` is directly used for conditional module loading in
+      // `ConditionalModule.registerWhen` as it does not support ConfigService injection. The purpose of
+      // keeping `metrics.enabled` in the configuration is for other modules to use and maintain consistency.
+      enabled: process.env.METRICS_ENABLED || "no",
+      config: jsonConfigMap.metricsConfig,
+    },
     registerDoiUri: process.env.REGISTER_DOI_URI,
     registerMetadataUri: process.env.REGISTER_METADATA_URI,
     doiUsername: process.env.DOI_USERNAME,

src/main.ts

@@ -15,9 +15,7 @@ async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     bufferLogs: true,
   });
-  const configService: ConfigService<Record<string, unknown>, false> = app.get(
-    ConfigService,
-  );
+  const configService = app.get(ConfigService);
   const apiVersion = configService.get<string>("versions.api");
   const swaggerPath = `${configService.get<string>("swaggerPath")}`;
 

src/metrics/metrics.module.ts

@@ -0,0 +1,34 @@
+import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common";
+import { ConfigModule, ConfigService } from "@nestjs/config";
+import { AccessTrackingMiddleware } from "./middlewares/accessTracking.middleware";
+import { JwtModule } from "@nestjs/jwt";
+
+@Module({
+  imports: [ConfigModule, JwtModule],
+  exports: [],
+})
+export class MetricsModule implements NestModule {
+  constructor(private readonly configService: ConfigService) {}
+
+  configure(consumer: MiddlewareConsumer) {
+    const { include = [], exclude = [] } =
+      this.configService.get("metrics.config") || {};
+    if (!include.length && !exclude.length) {
+      Logger.error(
+        'Metrics middleware requires at least one "include" or "exclude" path in the metricsConfig.json file.',
+        "MetricsModule",
+      );
+      return;
+    }
+
+    try {
+      consumer
+        .apply(AccessTrackingMiddleware)
+        .exclude(...exclude)
+        .forRoutes(...include);
+      Logger.log("Start collecting metrics", "MetricsModule");
+    } catch (error) {
+      Logger.error("Error configuring metrics middleware", error);
+    }
+  }
+}

src/metrics/middlewares/accessTracking.middleware.ts

@@ -0,0 +1,76 @@
+import { Injectable, Logger, NestMiddleware } from "@nestjs/common";
+import { Request, Response, NextFunction } from "express";
+import { JwtService } from "@nestjs/jwt";
+import { parse } from "url";
+
+@Injectable()
+export class AccessTrackingMiddleware implements NestMiddleware {
+  private requestCache = new Map<string, number>(); // Cache to store recent requests
+  private logIntervalDuration = 1000; // Log every 1 second to prevent spam
+  private cacheResetInterval = 10 * 60 * 1000; // Clear cache every 10 minutes to prevent memory leak
+
+  constructor(private readonly jwtService: JwtService) {
+    this.startCacheResetInterval();
+  }
+  use(req: Request, res: Response, next: NextFunction) {
+    const { query, pathname } = parse(req.originalUrl, true);
+
+    const userAgent = req.headers["user-agent"];
+    // TODO: Better to use a library for this?
+    const isBot = userAgent ? /bot|crawl|spider|slurp/i.test(userAgent) : false;
+
+    if (!pathname || isBot) return;
+
+    const startTime = Date.now();
+    const authHeader = req.headers.authorization;
+    const originIp = req.socket.remoteAddress;
+    const userId = this.parseToken(authHeader);
+
+    const cacheKeyIdentifier = `${userId}-${originIp}-${pathname}`;
+
+    res.on("finish", () => {
+      const statusCode = res.statusCode;
+      if (statusCode === 304) return;
+
+      const responseTime = Date.now() - startTime;
+
+      const lastHitTime = this.requestCache.get(cacheKeyIdentifier);
+
+      // Log only if the request was not recently logged
+      if (!lastHitTime || Date.now() - lastHitTime > this.logIntervalDuration) {
+        Logger.log("SciCatAccessLogs", {
+          userId,
+          originIp,
+          endpoint: pathname,
+          query: query,
+          statusCode,
+          responseTime,
+        });
+
+        this.requestCache.set(cacheKeyIdentifier, Date.now());
+      }
+    });
+
+    next();
+  }
+
+  private parseToken(authHeader?: string) {
+    if (!authHeader) return "anonymous";
+    const token = authHeader.split(" ")[1];
+    if (!token) return "anonymous";
+
+    try {
+      const { id } = this.jwtService.decode(token);
+      return id;
+    } catch (error) {
+      Logger.error("Error parsing token-> AccessTrackingMiddleware", error);
+      return null;
+    }
+  }
+
+  private startCacheResetInterval() {
+    setInterval(() => {
+      this.requestCache.clear();
+    }, this.cacheResetInterval);
+  }
+}