SciCatProject
diff --git a/‎.gitignore
Lines changed: 1 addition & 0 deletions b/‎.gitignore
Lines changed: 1 addition & 0 deletions
diff --git a/‎loggers.json.example
Lines changed: 18 additions & 0 deletions b/‎loggers.json.example
Lines changed: 18 additions & 0 deletions
diff --git a/‎package-lock.json
Lines changed: 1 addition & 0 deletions b/‎package-lock.json
Lines changed: 1 addition & 0 deletions
diff --git a/‎package.json
Lines changed: 1 addition & 0 deletions b/‎package.json
Lines changed: 1 addition & 0 deletions
diff --git a/‎src/app.module.ts
Lines changed: 2 additions & 0 deletions b/‎src/app.module.ts
Lines changed: 2 additions & 0 deletions
diff --git a/‎src/common/utils.ts
Lines changed: 4 additions & 1 deletion b/‎src/common/utils.ts
Lines changed: 4 additions & 1 deletion
diff --git a/‎src/config/configuration.ts
Lines changed: 29 additions & 2 deletions b/‎src/config/configuration.ts
Lines changed: 29 additions & 2 deletions
diff --git a/‎src/elastic-search/elastic-search.service.ts
Lines changed: 28 additions & 26 deletions b/‎src/elastic-search/elastic-search.service.ts
Lines changed: 28 additions & 26 deletions
diff --git a/‎src/loggers/interfaces/grayLoggerConfig.interface.ts
Lines changed: 7 additions & 0 deletions b/‎src/loggers/interfaces/grayLoggerConfig.interface.ts
Lines changed: 7 additions & 0 deletions
diff --git a/‎src/loggers/interfaces/logger.interface.ts
Lines changed: 17 additions & 0 deletions b/‎src/loggers/interfaces/logger.interface.ts
Lines changed: 17 additions & 0 deletions
@@ -2,6 +2,7 @@
 /dist
 /node_modules
 functionalAccounts.json
+loggers.json
 
 # Configs
 .env
 
@@ -0,0 +1,18 @@
+[
+  {
+    "type": "DefaultLogger",
+    "modulePath": "./loggingProviders/defaultLogger",
+    "config": {}
+  },
+  {
+    "type": "GrayLogger",
+    "modulePath": "./loggingProviders/grayLogger",
+    "config": {
+      "facility": "scicatOfficial",
+      "environment": "test",
+      "server": "localhost",
+      "service": "scicat-backend-next",
+      "port": 9999
+    }
+  }
+]
@@ -45,6 +45,7 @@
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/swagger": "^7.1.7",
     "@nestjs/terminus": "^10.1.1",
+    "@user-office-software/duo-logger": "^2.1.1",
     "@user-office-software/duo-message-broker": "^1.4.0",
     "bcrypt": "^5.1.0",
     "class-transformer": "^0.5.1",
 
@@ -31,6 +31,7 @@ import { CommonModule } from "./common/common.module";
 import { EventEmitterModule } from "@nestjs/event-emitter";
 import { AdminModule } from "./admin/admin.module";
 import { HealthModule } from "./health/health.module";
+import { LoggerModule } from "./loggers/logger.module";
 
 @Module({
   imports: [
@@ -41,6 +42,7 @@ import { HealthModule } from "./health/health.module";
     ConfigModule.forRoot({
       load: [configuration],
     }),
+    LoggerModule,
     DatablocksModule,
     DatasetsModule,
     InitialDatasetsModule,
 
@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/quotes */
 import { Logger } from "@nestjs/common";
+import { inspect } from "util";
 import { DateTime } from "luxon";
 import { format, unit } from "mathjs";
 import { Expression, FilterQuery, Model, PipelineStage } from "mongoose";
@@ -211,7 +212,9 @@ export const handleAxiosRequestError = (
     // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
     // http.ClientRequest in node.js
     console.error(error.request);
-    Logger.error({ request: error.request }, context);
+
+    // Inspect is needed to deal with circular references issue.
+    Logger.error({ request: inspect(error.request) }, context);
   } else {
     // Something happened in setting up the request that triggered an Error
     Logger.error("Error: " + error.message, context);
 
@@ -1,4 +1,4 @@
-import { Logger } from "@nestjs/common";
+import * as fs from "fs";
 
 const configuration = () => {
   const accessGroupsStaticValues =
@@ -22,10 +22,34 @@ const configuration = () => {
   const proposalGroups = process.env.PROPOSAL_GROUPS || ("" as string);
   const sampleGroups = process.env.SAMPLE_GROUPS || ("" as string);
 
+  const defaultLogger = {
+    type: "DefaultLogger",
+    modulePath: "./loggingProviders/defaultLogger",
+    config: {},
+  };
+  const jsonConfigMap: { [key: string]: object[] | boolean } = {};
+  const jsonConfigFileList: { [key: string]: string } = {
+    loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json",
+  };
+  Object.keys(jsonConfigFileList).forEach((key) => {
+    const filePath = jsonConfigFileList[key];
+    if (fs.existsSync(filePath)) {
+      const data = fs.readFileSync(filePath, "utf8");
+      try {
+        jsonConfigMap[key] = JSON.parse(data);
+      } catch (error) {
+        console.error(
+          "Error json config file parsing " + filePath + " : " + error,
+        );
+        jsonConfigMap[key] = false;
+      }
+    }
+  });
+
   // Logger.log("Config SETUP");
   // Logger.log("- Access groups statisc values : " + accessGroupsStaticValues);
   // Logger.log("- Admin groups : " + adminGroups);
-  // Logger.log("- Delete groups : " + deleteGroups);
+  // Logger.log("- Delete groups : " + deleteGroups );
   // Logger.log("- Create dataset groups : " + createDatasetGroups);
   // Logger.log(
   //   "- Create dataset with pid groups : " + createDatasetWithPidGroups,
@@ -37,6 +61,7 @@ const configuration = () => {
   // Logger.log("- Update job groups : " + updateJobGroups);
 
   return {
+    loggerConfigs: jsonConfigMap.loggers || [defaultLogger],
     adminGroups: adminGroups.split(",").map((v) => v.trim()) ?? [],
     deleteGroups: deleteGroups.split(",").map((v) => v.trim()) ?? [],
     createDatasetGroups: createDatasetGroups.split(",").map((v) => v.trim()),
@@ -81,6 +106,7 @@ const configuration = () => {
         usernameAttr: process.env.LDAP_USERNAME ?? "displayName",
       },
     },
+
     oidc: {
       issuer: process.env.OIDC_ISSUER, // Example: https://identity.esss.dk/realm/ess
       clientID: process.env.OIDC_CLIENT_ID, // Example: scicat
@@ -101,6 +127,7 @@ const configuration = () => {
       baseUrl:
         process.env.LOGBOOK_BASE_URL ?? "http://localhost:3030/scichatapi",
     },
+
     metadataKeysReturnLimit: process.env.METADATA_KEYS_RETURN_LIMIT
       ? parseInt(process.env.METADATA_KEYS_RETURN_LIMIT, 10)
       : undefined,
 
@@ -71,6 +71,7 @@ export class ElasticSearchService implements OnModuleInit {
     ) {
       Logger.error(
         "Missing ENVIRONMENT variables for elastic search connection",
+        "ElasticSearch",
       );
     }
   }
@@ -88,9 +89,9 @@ export class ElasticSearchService implements OnModuleInit {
         await this.createIndex(this.defaultIndex);
       }
       this.connected = true;
-      Logger.log("Elasticsearch Connected");
+      Logger.log("Elasticsearch Connected", "ElasticSearch");
     } catch (error) {
-      Logger.error("onModuleInit failed-> ElasticSearchService", error);
+      Logger.error(error, "onModuleInit failed-> ElasticSearchService");
     }
   }
 
@@ -161,10 +162,12 @@ export class ElasticSearchService implements OnModuleInit {
       await this.esService.indices.open({
         index,
       });
-      Logger.log(`Elasticsearch Index Created-> Index: ${index}`);
+      Logger.log(
+        `Elasticsearch Index Created-> Index: ${index}`,
+        "Elasticsearch",
+      );
       return HttpStatus.CREATED;
     } catch (error) {
-      Logger.error("createIndex failed-> ElasticSearchService", error);
       throw new HttpException(
         `createIndex failed-> ElasticSearchService ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -191,7 +194,6 @@ export class ElasticSearchService implements OnModuleInit {
     try {
       return await this.esService.count({ index });
     } catch (error) {
-      Logger.error("getCount failed-> ElasticSearchService", error);
       throw new HttpException(
         `getCount failed-> ElasticSearchService ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -221,9 +223,11 @@ export class ElasticSearchService implements OnModuleInit {
       await this.esService.indices.open({
         index,
       });
-      Logger.log(`Elasticsearch Index Updated-> Index: ${index}`);
+      Logger.log(
+        `Elasticsearch Index Updated-> Index: ${index}`,
+        "Elasticsearch",
+      );
     } catch (error) {
-      Logger.error("updateIndex failed-> ElasticSearchService", error);
       throw new HttpException(
         `updateIndex failed-> ElasticSearchService ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -235,7 +239,6 @@ export class ElasticSearchService implements OnModuleInit {
     try {
       return await this.esService.indices.getSettings({ index });
     } catch (error) {
-      Logger.error("getIndexSettings failed-> ElasticSearchService", error);
       throw new HttpException(
         `getIndexSettings failed-> ElasticSearchService ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -246,10 +249,12 @@ export class ElasticSearchService implements OnModuleInit {
   async deleteIndex(index = this.defaultIndex) {
     try {
       await this.esService.indices.delete({ index });
-      Logger.log(`Elasticsearch Index Deleted-> Index: ${index} `);
+      Logger.log(
+        `Elasticsearch Index Deleted-> Index: ${index} `,
+        "Elasticsearch",
+      );
       return { success: true, message: `Index ${index} deleted` };
     } catch (error) {
-      Logger.error("deleteIndex failed-> ElasticSearchService", error);
       throw new HttpException(
         `deleteIndex failed-> ElasticSearchService ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -323,8 +328,6 @@ export class ElasticSearchService implements OnModuleInit {
         data,
       };
     } catch (error) {
-      Logger.error("SearchService || search query issue || -> search", error);
-
       throw new HttpException(
         `SearchService || search query issue || -> search ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -352,11 +355,6 @@ export class ElasticSearchService implements OnModuleInit {
 
       return transformedFacets;
     } catch (error) {
-      Logger.error(
-        "SearchService || aggregate query issue || -> aggregate",
-        error,
-      );
-
       throw new HttpException(
         `SearchService || aggregate query issue || -> aggregate ${error}`,
         HttpStatus.BAD_REQUEST,
@@ -383,10 +381,14 @@ export class ElasticSearchService implements OnModuleInit {
       });
 
       Logger.log(
-        `Elasticsearch Document Update/inserted-> Document_id: ${data.pid} update/inserted on index: ${this.defaultIndex}`,
+        `Document Update/inserted-> Document_id: ${data.pid} update/inserted on index: ${this.defaultIndex}`,
+        "Elasticsearch",
       );
     } catch (error) {
-      Logger.error("updateDocument failed-> ElasticSearchService", error);
+      throw new HttpException(
+        `updateDocument failed-> ElasticSearchService ${error}`,
+        HttpStatus.BAD_REQUEST,
+      );
     }
   }
 
@@ -398,10 +400,14 @@ export class ElasticSearchService implements OnModuleInit {
         refresh: this.refresh,
       });
       Logger.log(
-        `Elasticsearch Document Deleted-> Document_id: ${id} deleted on index: ${this.defaultIndex}`,
+        `Document Deleted-> Document_id: ${id} deleted on index: ${this.defaultIndex}`,
+        "Elasticsearch",
       );
     } catch (error) {
-      Logger.error("deleteDocument failed-> ElasticSearchService", error);
+      throw new HttpException(
+        `deleteDocument failed-> ElasticSearchService ${error}`,
+        HttpStatus.BAD_REQUEST,
+      );
     }
   }
 
@@ -424,11 +430,7 @@ export class ElasticSearchService implements OnModuleInit {
         ];
       },
       onDrop(doc) {
-        console.error(doc.document._id, doc.error?.reason);
-        Logger.error(
-          "SearchService-> performBulkOperation-> data insert faield: ",
-          doc.document._id,
-        );
+        console.debug(`${doc.document._id}`, doc.error?.reason);
       },
     });
   }
 
@@ -0,0 +1,7 @@
+export interface GrayLoggerConfig {
+  server: string;
+  port: number;
+  facility: string;
+  environment: string;
+  service: string;
+}
@@ -0,0 +1,17 @@
+export interface Logger {
+  log(message: string, context: Record<string, unknown>): void;
+  error(message: string, context: Record<string, unknown>): void;
+  warn(message: string, context: Record<string, unknown>): void;
+  debug(message: string, context: Record<string, unknown>): void;
+  exception(
+    message: string,
+    exception: unknown,
+    context: Record<string, unknown>,
+  ): void;
+}
+
+export interface LoggerConfig {
+  type: string;
+  modulePath: string;
+  config?: Record<string, unknown>;
+}