Merge pull request #34 from robert-cronin/feature/odr_monitoring

Feature: add ODR monitoring module

.gitignore

@@ -58,6 +58,7 @@ node_modules/
 .coverage
 .htmlcov/
 .tox/
+coverage.xml
 
 # Packaging
 *.tar.gz

Taskfile.yml

@@ -7,6 +7,7 @@ includes:
   core: ./modules/odr_core/Taskfile.yml
   data: ./modules/odr_datamodel/Taskfile.yml
   api-test: ./modules/odr_api/tests/Taskfile.yml
+  monitoring: ./modules/odr_monitoring/Taskfile.yml
 
 dotenv: ['.env', '{{.ENV}}/.env', '{{.HOME}}/.env']
 vars:
@@ -75,6 +76,7 @@ tasks:
         task core:test-db &
         task db:test-postgres &
         task api-test:test-all &
+        task monitoring:test &
         wait
       - echo "All testing complete"
 

modules/odr_monitoring/README.md

@@ -0,0 +1,87 @@
+# ODR Monitoring
+
+ODR Monitoring is a logging and monitoring module for the Open Data Repository project. It provides a configurable logging system with custom formatters and handlers for both console and file output.
+
+## Installation
+
+To install the ODR Monitoring module, run:
+
+```
+pip install -e modules/odr_monitoring
+```
+
+## Usage
+
+To use the ODR Monitoring logger in your code:
+
+```python
+from odr_monitoring import get_logger
+
+logger = get_logger(__name__)
+
+logger.info("This is an info message")
+logger.error("This is an error message")
+```
+
+## Configuration
+
+The Open Data Repository (ODR) Monitoring logging system can be configured using environment variables or a .env file. The following configuration options are available:
+
+- `ODR_MONITORING_LOG_LEVEL`: The log level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Default: DEBUG
+- `ODR_MONITORING_LOG_FORMAT`: The log format string. Default: "%(levelname)s | %(asctime)s | %(message)s"
+- `ODR_MONITORING_LOG_DATE_FORMAT`: The date format string for log messages. Default: "%Y-%m-%d %H:%M:%S"
+- `ODR_MONITORING_CONSOLE_LOG_ENABLED`: Enable console logging. Default: True
+- `ODR_MONITORING_FILE_LOG_ENABLED`: Enable file logging. Default: True
+- `ODR_MONITORING_LOG_FILE_PATH`: The path to the log file. Default: "logs/odr_monitoring.log"
+- `ODR_MONITORING_LOG_FILE_MAX_BYTES`: The maximum size of the log file before rotation. Default: 10485760 (10 MB)
+- `ODR_MONITORING_LOG_FILE_BACKUP_COUNT`: The number of backup log files to keep. Default: 5
+- `ODR_MONITORING_USE_COLORS`: Enable colored output for console logging. Default: True
+
+To configure these options:
+
+1. Create a `.env` file in the root directory of your project.
+2. Add the desired configuration options to the `.env` file. For example:
+
+   ```
+   ODR_MONITORING_LOG_LEVEL=INFO
+   ODR_MONITORING_FILE_LOG_ENABLED=False
+   ODR_MONITORING_USE_COLORS=False
+   ```
+
+## Development
+
+To set up the development environment:
+
+1. Clone the repository
+2. Install the requirements: `pip install -r requirements.txt`
+3. Install the pre-commit hooks: `pre-commit install`
+
+To run the tests:
+
+```
+task test
+```
+
+To run the tests with coverage:
+
+```
+task coverage
+```
+
+To run the linter:
+
+```
+task lint
+```
+
+To format the code:
+
+```
+task format
+```
+
+To run all checks:
+
+```
+task check
+```

modules/odr_monitoring/Taskfile.yml

@@ -0,0 +1,48 @@
+version: "3"
+
+tasks:
+  default:
+    desc: List all the tasks
+    cmds:
+      - task --list
+
+  install:
+    desc: Install the package in editable mode
+    dir: modules/odr_monitoring
+    cmds:
+      - pip install -e .
+
+  test:
+    desc: Run tests for the monitoring module
+    deps: [install]
+    dir: modules/odr_monitoring
+    cmds:
+      - python -m pytest tests
+
+  coverage:
+    desc: Run tests with coverage report
+    deps: [install]
+    dir: modules/odr_monitoring
+    cmds:
+      - python -m pytest --cov=odr_monitoring --cov-report=term-missing --cov-report=xml tests
+
+  lint:
+    desc: Run linter on the monitoring module
+    dir: modules/odr_monitoring
+    cmds:
+      - flake8 .
+
+  format:
+    desc: Format the code using black
+    dir: modules/odr_monitoring
+    cmds:
+      - black .
+
+  check:
+    desc: Run all checks (tests, lint, format)
+    dir: modules/odr_monitoring
+    cmds:
+      - task: install
+      - task: test
+      - task: lint
+      - task: format

modules/odr_monitoring/odr_monitoring/__init__.py

@@ -0,0 +1,3 @@
+from .logger import get_logger
+
+__all__ = ["get_logger"]

modules/odr_monitoring/odr_monitoring/config.py

@@ -0,0 +1,21 @@
+from pydantic_settings import BaseSettings
+from pydantic import ConfigDict
+
+
+class LoggingConfig(BaseSettings):
+    LOG_LEVEL: str = "DEBUG"
+    LOG_FORMAT: str = "%(levelname)s | %(asctime)s | %(message)s"
+    LOG_DATE_FORMAT: str = "%Y-%m-%d %H:%M:%S"
+    CONSOLE_LOG_ENABLED: bool = True
+    FILE_LOG_ENABLED: bool = True
+    LOG_FILE_PATH: str = "logs/odr_monitoring.log"
+    LOG_FILE_MAX_BYTES: int = 10 * 1024 * 1024  # 10 MB
+    LOG_FILE_BACKUP_COUNT: int = 5
+    USE_COLORS: bool = True
+
+    model_config = ConfigDict(env_prefix="ODR_MONITORING_", env_file=".env")
+
+
+logging_config = LoggingConfig()
+
+__all__ = ['logging_config']

modules/odr_monitoring/odr_monitoring/formatters/standard_formatter.py

@@ -0,0 +1,37 @@
+import logging
+import json
+
+
+class StandardFormatter(logging.Formatter):
+    COLORS = {
+        'DEBUG': '\033[94m',        # Blue
+        'INFO': '\033[92m',         # Green
+        'WARNING': '\033[93m',      # Yellow
+        'ERROR': '\033[91m',        # Red
+        'CRITICAL': '\033[95m',     # Magenta
+        'RESET': '\033[0m'          # Reset color
+    }
+
+    def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, use_colors=True):
+        super().__init__(fmt, datefmt, style, validate)
+        self.use_colors = use_colors
+
+    def format(self, record):
+        log_data = {
+            'timestamp': self.formatTime(record, self.datefmt),
+            'level': record.levelname,
+            'message': record.getMessage(),
+            'module': record.module,
+            'function': record.funcName,
+            'line': record.lineno,
+        }
+        if record.exc_info:
+            log_data['exception'] = self.formatException(record.exc_info)
+
+        if self.use_colors:
+            color = self.COLORS.get(record.levelname, self.COLORS['RESET'])
+            formatted_message = f"{color}{json.dumps(log_data)}{self.COLORS['RESET']}"
+        else:
+            formatted_message = json.dumps(log_data)
+
+        return formatted_message

modules/odr_monitoring/odr_monitoring/handlers/__init__.py

@@ -0,0 +1,4 @@
+from .console_handler import get_console_handler
+from .file_handler import get_file_handler
+
+__all__ = ['get_console_handler', 'get_file_handler']

modules/odr_monitoring/odr_monitoring/handlers/console_handler.py

@@ -0,0 +1,19 @@
+import logging
+import sys
+from odr_monitoring.config import logging_config
+from odr_monitoring.formatters.standard_formatter import StandardFormatter
+
+
+class ConsoleHandler(logging.StreamHandler):
+    def __init__(self):
+        super().__init__(stream=sys.stdout)
+        self.setLevel(logging.getLevelName(logging_config.LOG_LEVEL))
+        self.setFormatter(StandardFormatter(
+            fmt=logging_config.LOG_FORMAT,
+            datefmt=logging_config.LOG_DATE_FORMAT,
+            use_colors=logging_config.USE_COLORS
+        ))
+
+
+def get_console_handler():
+    return ConsoleHandler()

modules/odr_monitoring/odr_monitoring/handlers/file_handler.py

@@ -0,0 +1,26 @@
+import os
+import logging
+from logging.handlers import RotatingFileHandler
+from odr_monitoring.config import logging_config
+from odr_monitoring.formatters.standard_formatter import StandardFormatter
+
+
+class FileHandler(RotatingFileHandler):
+    def __init__(self):
+        log_dir = os.path.dirname(logging_config.LOG_FILE_PATH)
+        os.makedirs(log_dir, exist_ok=True)
+        super().__init__(
+            filename=logging_config.LOG_FILE_PATH,
+            maxBytes=logging_config.LOG_FILE_MAX_BYTES,
+            backupCount=logging_config.LOG_FILE_BACKUP_COUNT
+        )
+        self.setLevel(logging.DEBUG)
+        self.setFormatter(StandardFormatter(
+            fmt=logging_config.LOG_FORMAT,
+            datefmt=logging_config.LOG_DATE_FORMAT,
+            use_colors=False
+        ))
+
+
+def get_file_handler():
+    return FileHandler()

modules/odr_monitoring/odr_monitoring/logger.py

@@ -0,0 +1,62 @@
+import logging
+import json
+from odr_monitoring.config import logging_config
+from odr_monitoring.handlers import get_console_handler, get_file_handler
+
+
+class ODRLogger:
+    def __init__(self, name):
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(logging.DEBUG)
+
+        self.setup_handlers()
+
+    def setup_handlers(self):
+        if logging_config.CONSOLE_LOG_ENABLED:
+            self.logger.addHandler(get_console_handler())
+
+        if logging_config.FILE_LOG_ENABLED:
+            self.logger.addHandler(get_file_handler())
+
+    def debug(self, msg, *args, **kwargs):
+        self.logger.debug(msg, *args, **kwargs)
+
+    def info(self, msg, *args, **kwargs):
+        self.logger.info(msg, *args, **kwargs)
+
+    def warning(self, msg, *args, **kwargs):
+        self.logger.warning(msg, *args, **kwargs)
+
+    def error(self, msg, *args, **kwargs):
+        self.logger.error(msg, *args, **kwargs)
+
+    def critical(self, msg, *args, **kwargs):
+        self.logger.critical(msg, *args, **kwargs)
+
+    def log_api_error(self, error: Exception, request_data: dict = None, additional_info: str = None):
+        error_message = f"API Error: {str(error)}"
+        if request_data:
+            error_message += f"\nRequest Data: {json.dumps(request_data, indent=2)}"
+        if additional_info:
+            error_message += f"\nAdditional Info: {additional_info}"
+        self.error(error_message)
+
+    def log_api_request(self, method: str, url: str, status_code: int, request_data: dict = None, response_data: dict = None):
+        log_message = f"API Request: {method} {url} - Status: {status_code}"
+        if request_data:
+            log_message += f"\nRequest Data: {json.dumps(request_data, indent=2)}"
+        if response_data:
+            log_message += f"\nResponse Data: {json.dumps(response_data, indent=2)}"
+        self.info(log_message)
+
+    def close(self):
+        for handler in self.logger.handlers[:]:
+            handler.close()
+            self.logger.removeHandler(handler)
+
+
+def get_logger(name):
+    return ODRLogger(name)
+
+
+__all__ = ['get_logger']

modules/odr_monitoring/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = [
+  "setuptools>=69.0",
+  "wheel",
+]
+
+[project]
+description = "This module contains logic for open data repository monitoring and logging."
+name = "odr_monitoring"
+version = "0.1.0"
+dynamic = ["dependencies"]
+
+[tool.setuptools.dynamic]
+dependencies = { file = ["requirements.txt"] }
+
+[tool.pytest.ini_options]
+pythonpath = [
+  "odr_monitoring",
+]
+
+[tool.setuptools.packages.find]
+include = ["odr_monitoring", "odr_monitoring.*"] # include the .xslt files
+exclude = []  # exclude packages matching these glob patterns (empty by default)
+
+[tool.setuptools.package-data]
+"odr_monitoring" = ["py.typed"]

modules/odr_monitoring/requirements.txt

@@ -0,0 +1,5 @@
+pydantic==2.8.2
+black==23.7.0
+flake8==7.1.1
+pytest==8.3.2
+pytest-cov==5.0.0