Better logging

mars-cli/mars_cli.py

@@ -1,25 +1,23 @@
 import click
-import logging
-import pathlib
-from configparser import ConfigParser
 from datetime import datetime
 from mars_lib.target_repo import TargetRepository
 from mars_lib.models.isa_json import IsaJson
 from mars_lib.submit import submission
 from mars_lib.credential import CredentialManager
-from mars_lib.logging import print_and_log
+from mars_lib.logging import print_and_log, init_logging
 from mars_lib.validation import validate, CustomValidationException
-from logging.handlers import RotatingFileHandler
 import requests
 import sys
-import os
 import json
+from pathlib import Path
+import os
+from configparser import ConfigParser
 
 # Load CLI configuration
 home_dir = (
-    pathlib.Path(str(os.getenv("MARS_SETTINGS_DIR")))
+    Path(str(os.getenv("MARS_SETTINGS_DIR")))
     if os.getenv("MARS_SETTINGS_DIR")
-    else pathlib.Path.home()
+    else Path.home()
 )
 
 config_file = home_dir / ".mars" / "settings.ini"
@@ -28,25 +26,8 @@
 config = ConfigParser()
 config.read(config_file)
 
-# Logging configuration
-log_level = config.get("logging", "log_level", fallback="ERROR")
-log_file = config.get("logging", "log_file", fallback=fallback_log_file)
-log_max_size = int(
-    config.get("logging", "log_max_size", fallback="1024")
-)  # in kilobytes. 1 MB by default.
-log_max_files = int(
-    config.get("logging", "log_max_files", fallback="5")
-)  # number of backup files. 5 by default.
-
-handler = RotatingFileHandler(
-    log_file, maxBytes=log_max_size * 1024, backupCount=log_max_files
-)
-handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
-
-logging.basicConfig(
-    handlers=[handler],
-    level=log_level,
-)
+# Load logging configuration
+init_logging(config, fallback_log_file)
 
 # Read in all the URLs from the config file
 urls = TargetRepository.get_repository_urls_from_config(config)
@@ -62,6 +43,10 @@
 @click.pass_context
 def cli(ctx, development):
     print_and_log("############# Welcome to the MARS CLI. #############")
+    print_and_log(
+        "Sensitive information might be dumped in the log files when setting the 'log_level' to DEBUG in the config file. Logging in debug mode should only be used for developing purpose a can implicate security issues if used in a production environment!",
+        "debug",
+    )
     print_and_log(
         f"Running in {'Development environment' if development else 'Production environment'}"
     )

mars-cli/mars_lib/logging.py

@@ -1,18 +1,54 @@
-import click
 import logging
-import sys
+import colorlog
+from configparser import ConfigParser
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+logger = logging.getLogger("MARS-CLI")
 
 
 def print_and_log(msg, level="info"):
     if level == "info":
-        click.echo(msg)
-        logging.info(msg)
-    elif level == "error":
-        click.echo(msg, file=sys.stderr)
-        logging.error(msg)
+        logger.info(msg)
     elif level == "warning":
-        click.echo(msg)
-        logging.warning(msg)
+        logger.warning(msg)
+    elif level == "error":
+        logger.error(msg)
+    elif level == "critical":
+        logger.critical(msg)
     else:
-        click.echo(msg)
-        logging.debug(msg)
+        logger.debug(msg)
+
+
+def init_logging(config: ConfigParser, fallback_log_file: Path) -> None:
+    # Logging configuration
+    log_level = config.get("logging", "log_level", fallback="ERROR")
+    log_file = config.get("logging", "log_file", fallback=fallback_log_file)
+    log_max_size = int(
+        config.get("logging", "log_max_size", fallback="1024")
+    )  # in kilobytes. 1 MB by default.
+    log_max_files = int(
+        config.get("logging", "log_max_files", fallback="5")
+    )  # number of backup files. 5 by default.
+    file_handler = RotatingFileHandler(
+        log_file, maxBytes=log_max_size * 1024, backupCount=log_max_files
+    )
+    file_handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+
+    stream_handler = colorlog.StreamHandler()
+    stream_handler.setFormatter(
+        colorlog.ColoredFormatter(
+            "%(log_color)s%(asctime)s - %(name)s - %(levelname)s - %(message)s%(reset)s",
+            log_colors={
+                "DEBUG": "bold_cyan",
+                "INFO": "green",
+                "WARNING": "yellow",
+                "ERROR": "red",
+                "CRITICAL": "red,bg_white",
+            },
+            style="%",
+        )
+    )
+    logging.basicConfig(handlers=[file_handler, stream_handler], level=log_level)

mars-cli/requirements.txt

@@ -4,3 +4,4 @@ keyring
 pydantic
 click
 retry
+colorlog