From a017ae8aa1dc08d55bddeafd9dbe74a65212a50b Mon Sep 17 00:00:00 2001 From: Robert Cronin Date: Wed, 28 Aug 2024 19:25:18 +1000 Subject: [PATCH] Feature: add odr_monitoring module Signed-off-by: Robert Cronin --- .gitignore | 1 + Taskfile.yml | 2 + modules/odr_monitoring/README.md | 87 +++++++++++++++++++ modules/odr_monitoring/Taskfile.yml | 48 ++++++++++ .../odr_monitoring/odr_monitoring/__init__.py | 3 + .../odr_monitoring/odr_monitoring/config.py | 21 +++++ .../odr_monitoring/formatters/__init__.py | 0 .../formatters/standard_formatter.py | 37 ++++++++ .../odr_monitoring/handlers/__init__.py | 4 + .../handlers/console_handler.py | 19 ++++ .../odr_monitoring/handlers/file_handler.py | 26 ++++++ .../odr_monitoring/odr_monitoring/logger.py | 62 +++++++++++++ modules/odr_monitoring/pyproject.toml | 27 ++++++ modules/odr_monitoring/requirements.txt | 5 ++ modules/odr_monitoring/tests/__init__.py | 0 modules/odr_monitoring/tests/test_logger.py | 56 ++++++++++++ 16 files changed, 398 insertions(+) create mode 100644 modules/odr_monitoring/README.md create mode 100644 modules/odr_monitoring/Taskfile.yml create mode 100644 modules/odr_monitoring/odr_monitoring/__init__.py create mode 100644 modules/odr_monitoring/odr_monitoring/config.py create mode 100644 modules/odr_monitoring/odr_monitoring/formatters/__init__.py create mode 100644 modules/odr_monitoring/odr_monitoring/formatters/standard_formatter.py create mode 100644 modules/odr_monitoring/odr_monitoring/handlers/__init__.py create mode 100644 modules/odr_monitoring/odr_monitoring/handlers/console_handler.py create mode 100644 modules/odr_monitoring/odr_monitoring/handlers/file_handler.py create mode 100644 modules/odr_monitoring/odr_monitoring/logger.py create mode 100644 modules/odr_monitoring/pyproject.toml create mode 100644 modules/odr_monitoring/requirements.txt create mode 100644 modules/odr_monitoring/tests/__init__.py create mode 100644 modules/odr_monitoring/tests/test_logger.py diff --git a/.gitignore b/.gitignore index 00a9121..e0e2a78 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ node_modules/ .coverage .htmlcov/ .tox/ +coverage.xml # Packaging *.tar.gz diff --git a/Taskfile.yml b/Taskfile.yml index f9f5575..a008f0d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -7,6 +7,7 @@ includes: core: ./modules/odr_core/Taskfile.yml data: ./modules/odr_datamodel/Taskfile.yml api-test: ./modules/odr_api/tests/Taskfile.yml + monitoring: ./modules/odr_monitoring/Taskfile.yml dotenv: ['.env', '{{.ENV}}/.env', '{{.HOME}}/.env'] vars: @@ -75,6 +76,7 @@ tasks: task core:test-db & task db:test-postgres & task api-test:test-all & + task monitoring:test & wait - echo "All testing complete" diff --git a/modules/odr_monitoring/README.md b/modules/odr_monitoring/README.md new file mode 100644 index 0000000..40d8717 --- /dev/null +++ b/modules/odr_monitoring/README.md @@ -0,0 +1,87 @@ +# ODR Monitoring + +ODR Monitoring is a logging and monitoring module for the Open Data Repository project. It provides a configurable logging system with custom formatters and handlers for both console and file output. + +## Installation + +To install the ODR Monitoring module, run: + +``` +pip install -e modules/odr_monitoring +``` + +## Usage + +To use the ODR Monitoring logger in your code: + +```python +from odr_monitoring import get_logger + +logger = get_logger(__name__) + +logger.info("This is an info message") +logger.error("This is an error message") +``` + +## Configuration + +The Open Data Repository (ODR) Monitoring logging system can be configured using environment variables or a .env file. The following configuration options are available: + +- `ODR_MONITORING_LOG_LEVEL`: The log level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Default: DEBUG +- `ODR_MONITORING_LOG_FORMAT`: The log format string. Default: "%(levelname)s | %(asctime)s | %(message)s" +- `ODR_MONITORING_LOG_DATE_FORMAT`: The date format string for log messages. Default: "%Y-%m-%d %H:%M:%S" +- `ODR_MONITORING_CONSOLE_LOG_ENABLED`: Enable console logging. Default: True +- `ODR_MONITORING_FILE_LOG_ENABLED`: Enable file logging. Default: True +- `ODR_MONITORING_LOG_FILE_PATH`: The path to the log file. Default: "logs/odr_monitoring.log" +- `ODR_MONITORING_LOG_FILE_MAX_BYTES`: The maximum size of the log file before rotation. Default: 10485760 (10 MB) +- `ODR_MONITORING_LOG_FILE_BACKUP_COUNT`: The number of backup log files to keep. Default: 5 +- `ODR_MONITORING_USE_COLORS`: Enable colored output for console logging. Default: True + +To configure these options: + +1. Create a `.env` file in the root directory of your project. +2. Add the desired configuration options to the `.env` file. For example: + + ``` + ODR_MONITORING_LOG_LEVEL=INFO + ODR_MONITORING_FILE_LOG_ENABLED=False + ODR_MONITORING_USE_COLORS=False + ``` + +## Development + +To set up the development environment: + +1. Clone the repository +2. Install the requirements: `pip install -r requirements.txt` +3. Install the pre-commit hooks: `pre-commit install` + +To run the tests: + +``` +task test +``` + +To run the tests with coverage: + +``` +task coverage +``` + +To run the linter: + +``` +task lint +``` + +To format the code: + +``` +task format +``` + +To run all checks: + +``` +task check +``` diff --git a/modules/odr_monitoring/Taskfile.yml b/modules/odr_monitoring/Taskfile.yml new file mode 100644 index 0000000..acf658c --- /dev/null +++ b/modules/odr_monitoring/Taskfile.yml @@ -0,0 +1,48 @@ +version: "3" + +tasks: + default: + desc: List all the tasks + cmds: + - task --list + + install: + desc: Install the package in editable mode + dir: modules/odr_monitoring + cmds: + - pip install -e . + + test: + desc: Run tests for the monitoring module + deps: [install] + dir: modules/odr_monitoring + cmds: + - python -m pytest tests + + coverage: + desc: Run tests with coverage report + deps: [install] + dir: modules/odr_monitoring + cmds: + - python -m pytest --cov=odr_monitoring --cov-report=term-missing --cov-report=xml tests + + lint: + desc: Run linter on the monitoring module + dir: modules/odr_monitoring + cmds: + - flake8 . + + format: + desc: Format the code using black + dir: modules/odr_monitoring + cmds: + - black . + + check: + desc: Run all checks (tests, lint, format) + dir: modules/odr_monitoring + cmds: + - task: install + - task: test + - task: lint + - task: format diff --git a/modules/odr_monitoring/odr_monitoring/__init__.py b/modules/odr_monitoring/odr_monitoring/__init__.py new file mode 100644 index 0000000..d6cb358 --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/__init__.py @@ -0,0 +1,3 @@ +from .logger import get_logger + +__all__ = ["get_logger"] diff --git a/modules/odr_monitoring/odr_monitoring/config.py b/modules/odr_monitoring/odr_monitoring/config.py new file mode 100644 index 0000000..267c7fb --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/config.py @@ -0,0 +1,21 @@ +from pydantic_settings import BaseSettings +from pydantic import ConfigDict + + +class LoggingConfig(BaseSettings): + LOG_LEVEL: str = "DEBUG" + LOG_FORMAT: str = "%(levelname)s | %(asctime)s | %(message)s" + LOG_DATE_FORMAT: str = "%Y-%m-%d %H:%M:%S" + CONSOLE_LOG_ENABLED: bool = True + FILE_LOG_ENABLED: bool = True + LOG_FILE_PATH: str = "logs/odr_monitoring.log" + LOG_FILE_MAX_BYTES: int = 10 * 1024 * 1024 # 10 MB + LOG_FILE_BACKUP_COUNT: int = 5 + USE_COLORS: bool = True + + model_config = ConfigDict(env_prefix="ODR_MONITORING_", env_file=".env") + + +logging_config = LoggingConfig() + +__all__ = ['logging_config'] diff --git a/modules/odr_monitoring/odr_monitoring/formatters/__init__.py b/modules/odr_monitoring/odr_monitoring/formatters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/odr_monitoring/odr_monitoring/formatters/standard_formatter.py b/modules/odr_monitoring/odr_monitoring/formatters/standard_formatter.py new file mode 100644 index 0000000..9955585 --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/formatters/standard_formatter.py @@ -0,0 +1,37 @@ +import logging +import json + + +class StandardFormatter(logging.Formatter): + COLORS = { + 'DEBUG': '\033[94m', # Blue + 'INFO': '\033[92m', # Green + 'WARNING': '\033[93m', # Yellow + 'ERROR': '\033[91m', # Red + 'CRITICAL': '\033[95m', # Magenta + 'RESET': '\033[0m' # Reset color + } + + def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, use_colors=True): + super().__init__(fmt, datefmt, style, validate) + self.use_colors = use_colors + + def format(self, record): + log_data = { + 'timestamp': self.formatTime(record, self.datefmt), + 'level': record.levelname, + 'message': record.getMessage(), + 'module': record.module, + 'function': record.funcName, + 'line': record.lineno, + } + if record.exc_info: + log_data['exception'] = self.formatException(record.exc_info) + + if self.use_colors: + color = self.COLORS.get(record.levelname, self.COLORS['RESET']) + formatted_message = f"{color}{json.dumps(log_data)}{self.COLORS['RESET']}" + else: + formatted_message = json.dumps(log_data) + + return formatted_message diff --git a/modules/odr_monitoring/odr_monitoring/handlers/__init__.py b/modules/odr_monitoring/odr_monitoring/handlers/__init__.py new file mode 100644 index 0000000..26fde8f --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/handlers/__init__.py @@ -0,0 +1,4 @@ +from .console_handler import get_console_handler +from .file_handler import get_file_handler + +__all__ = ['get_console_handler', 'get_file_handler'] diff --git a/modules/odr_monitoring/odr_monitoring/handlers/console_handler.py b/modules/odr_monitoring/odr_monitoring/handlers/console_handler.py new file mode 100644 index 0000000..2038ab0 --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/handlers/console_handler.py @@ -0,0 +1,19 @@ +import logging +import sys +from odr_monitoring.config import logging_config +from odr_monitoring.formatters.standard_formatter import StandardFormatter + + +class ConsoleHandler(logging.StreamHandler): + def __init__(self): + super().__init__(stream=sys.stdout) + self.setLevel(logging.getLevelName(logging_config.LOG_LEVEL)) + self.setFormatter(StandardFormatter( + fmt=logging_config.LOG_FORMAT, + datefmt=logging_config.LOG_DATE_FORMAT, + use_colors=logging_config.USE_COLORS + )) + + +def get_console_handler(): + return ConsoleHandler() diff --git a/modules/odr_monitoring/odr_monitoring/handlers/file_handler.py b/modules/odr_monitoring/odr_monitoring/handlers/file_handler.py new file mode 100644 index 0000000..3964c11 --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/handlers/file_handler.py @@ -0,0 +1,26 @@ +import os +import logging +from logging.handlers import RotatingFileHandler +from odr_monitoring.config import logging_config +from odr_monitoring.formatters.standard_formatter import StandardFormatter + + +class FileHandler(RotatingFileHandler): + def __init__(self): + log_dir = os.path.dirname(logging_config.LOG_FILE_PATH) + os.makedirs(log_dir, exist_ok=True) + super().__init__( + filename=logging_config.LOG_FILE_PATH, + maxBytes=logging_config.LOG_FILE_MAX_BYTES, + backupCount=logging_config.LOG_FILE_BACKUP_COUNT + ) + self.setLevel(logging.DEBUG) + self.setFormatter(StandardFormatter( + fmt=logging_config.LOG_FORMAT, + datefmt=logging_config.LOG_DATE_FORMAT, + use_colors=False + )) + + +def get_file_handler(): + return FileHandler() diff --git a/modules/odr_monitoring/odr_monitoring/logger.py b/modules/odr_monitoring/odr_monitoring/logger.py new file mode 100644 index 0000000..72e23a8 --- /dev/null +++ b/modules/odr_monitoring/odr_monitoring/logger.py @@ -0,0 +1,62 @@ +import logging +import json +from odr_monitoring.config import logging_config +from odr_monitoring.handlers import get_console_handler, get_file_handler + + +class ODRLogger: + def __init__(self, name): + self.logger = logging.getLogger(name) + self.logger.setLevel(logging.DEBUG) + + self.setup_handlers() + + def setup_handlers(self): + if logging_config.CONSOLE_LOG_ENABLED: + self.logger.addHandler(get_console_handler()) + + if logging_config.FILE_LOG_ENABLED: + self.logger.addHandler(get_file_handler()) + + def debug(self, msg, *args, **kwargs): + self.logger.debug(msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + self.logger.info(msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + self.logger.warning(msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + self.logger.error(msg, *args, **kwargs) + + def critical(self, msg, *args, **kwargs): + self.logger.critical(msg, *args, **kwargs) + + def log_api_error(self, error: Exception, request_data: dict = None, additional_info: str = None): + error_message = f"API Error: {str(error)}" + if request_data: + error_message += f"\nRequest Data: {json.dumps(request_data, indent=2)}" + if additional_info: + error_message += f"\nAdditional Info: {additional_info}" + self.error(error_message) + + def log_api_request(self, method: str, url: str, status_code: int, request_data: dict = None, response_data: dict = None): + log_message = f"API Request: {method} {url} - Status: {status_code}" + if request_data: + log_message += f"\nRequest Data: {json.dumps(request_data, indent=2)}" + if response_data: + log_message += f"\nResponse Data: {json.dumps(response_data, indent=2)}" + self.info(log_message) + + def close(self): + for handler in self.logger.handlers[:]: + handler.close() + self.logger.removeHandler(handler) + + +def get_logger(name): + return ODRLogger(name) + + +__all__ = ['get_logger'] diff --git a/modules/odr_monitoring/pyproject.toml b/modules/odr_monitoring/pyproject.toml new file mode 100644 index 0000000..e3559b2 --- /dev/null +++ b/modules/odr_monitoring/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools>=69.0", + "wheel", +] + +[project] +description = "This module contains logic for open data repository monitoring and logging." +name = "odr_monitoring" +version = "0.1.0" +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = { file = ["requirements.txt"] } + +[tool.pytest.ini_options] +pythonpath = [ + "odr_monitoring", +] + +[tool.setuptools.packages.find] +include = ["odr_monitoring", "odr_monitoring.*"] # include the .xslt files +exclude = [] # exclude packages matching these glob patterns (empty by default) + +[tool.setuptools.package-data] +"odr_monitoring" = ["py.typed"] diff --git a/modules/odr_monitoring/requirements.txt b/modules/odr_monitoring/requirements.txt new file mode 100644 index 0000000..d83e144 --- /dev/null +++ b/modules/odr_monitoring/requirements.txt @@ -0,0 +1,5 @@ +pydantic==2.8.2 +black==23.7.0 +flake8==7.1.1 +pytest==8.3.2 +pytest-cov==5.0.0 diff --git a/modules/odr_monitoring/tests/__init__.py b/modules/odr_monitoring/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/modules/odr_monitoring/tests/test_logger.py b/modules/odr_monitoring/tests/test_logger.py new file mode 100644 index 0000000..3ae55df --- /dev/null +++ b/modules/odr_monitoring/tests/test_logger.py @@ -0,0 +1,56 @@ +import unittest +import os +import json +from odr_monitoring import get_logger +from odr_monitoring.config import logging_config + + +class TestODRLogger(unittest.TestCase): + def setUp(self): + self.logger = get_logger("test_logger") + self.log_file = logging_config.LOG_FILE_PATH + + def test_logger_creation(self): + self.assertIsNotNone(self.logger) + + def test_log_levels(self): + self.logger.debug("Debug message") + self.logger.info("Info message") + self.logger.warning("Warning message") + self.logger.error("Error message") + self.logger.critical("Critical message") + self.assertTrue(os.path.exists(self.log_file), f"Log file not created at {self.log_file}") + + with open(self.log_file, "r") as f: + log_contents = f.readlines() + + log_messages = [json.loads(line)['message'] for line in log_contents] + + self.assertIn("Debug message", log_messages) + self.assertIn("Info message", log_messages) + self.assertIn("Warning message", log_messages) + self.assertIn("Error message", log_messages) + self.assertIn("Critical message", log_messages) + + def test_log_format(self): + self.logger.info("Test message") + self.assertTrue(os.path.exists(self.log_file), f"Log file not created at {self.log_file}") + + with open(self.log_file, "r") as f: + last_log = json.loads(f.readlines()[-1]) + + self.assertIn("timestamp", last_log) + self.assertIn("level", last_log) + self.assertIn("message", last_log) + self.assertIn("module", last_log) + self.assertIn("function", last_log) + self.assertIn("line", last_log) + + def tearDown(self): + self.logger.close() + if os.path.exists(self.log_file): + os.remove(self.log_file) + + +if __name__ == "__main__": + unittest.main()