From 084d128393f93aff1780927af12e7e02d001b851 Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Fri, 22 Dec 2023 10:59:43 +0200 Subject: [PATCH 1/8] updated README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c612e14..19f13f8 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Supported versions: Tested on: -* Zabbix 5.0, 6.0, 6.4 and 7.0.0 alpha 7 +* Zabbix 5.0, 6.0, 6.4 and pre-7.0 * Python 3.8, 3.9, 3.10 and 3.11 ## Documentation From e9fb3c3c0c8f321798f3e85e78a1851ccf644717 Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Fri, 22 Dec 2023 11:06:00 +0200 Subject: [PATCH 2/8] fixed flaws in the __prepare_request annotation --- zabbix_utils/common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zabbix_utils/common.py b/zabbix_utils/common.py index 671ee56..9d8255c 100644 --- a/zabbix_utils/common.py +++ b/zabbix_utils/common.py @@ -126,22 +126,22 @@ class ZabbixProtocol(): HEADER_SIZE = 13 @classmethod - def __prepare_request(cls, data: Union[bytes, str, dict]) -> bytes: + def __prepare_request(cls, data: Union[bytes, str, list, dict]) -> bytes: if isinstance(data, bytes): return data if isinstance(data, str): return data.encode("utf-8") if isinstance(data, list) or isinstance(data, dict): return json.dumps(data, ensure_ascii=False).encode("utf-8") - raise TypeError("Unsupported data type, only 'bytes', 'str' or 'dict' is expected") + raise TypeError("Unsupported data type, only 'bytes', 'str', 'list' or 'dict' is expected") @classmethod - def create_packet(cls, payload: Union[bytes, str, dict], + def create_packet(cls, payload: Union[bytes, str, list, dict], log: Logger, compression: bool = False) -> bytes: """Create a packet for sending via the Zabbix protocol. Args: - payload (Union[bytes, str, dict]): Payload of the future packet + payload (Union[bytes, str, list, dict]): Payload of the future packet log (Logger): Logger object compression (bool, optional): Compression use flag. Defaults to `False`. From 258544b46ba66dc9684ee98bbe5eb3a1ea68c08c Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Fri, 22 Dec 2023 15:09:55 +0200 Subject: [PATCH 3/8] fixed issue with hiding private (sensitive) information --- tests/test_zabbix_api.py | 28 ++++++++++++++++++---------- zabbix_utils/api.py | 10 +++++----- zabbix_utils/common.py | 35 ++++++++++++++++++++++------------- zabbix_utils/exceptions.py | 7 ++++--- zabbix_utils/logger.py | 14 +++++++++----- 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/tests/test_zabbix_api.py b/tests/test_zabbix_api.py index b96fa1f..ad03a33 100644 --- a/tests/test_zabbix_api.py +++ b/tests/test_zabbix_api.py @@ -478,24 +478,32 @@ def test_hide_private(self): test_cases = [ { - 'input': json.dumps({"auth": "q2BTIw85kqmjtXl3","token": "jZAC51wHuWdwvQnx"}), - 'output': json.dumps({"auth": mask, "token": mask}) + 'input': {"auth": "q2BTIw85kqmjtXl3","token": "jZAC51wHuWdwvQnx"}, + 'output': {"auth": mask, "token": mask} }, { - 'input': json.dumps({"token": "jZAC51wHuWdwvQnxwbP2T55vh6R5R2uW"}), - 'output': json.dumps({"token": f"jZAC{mask}R2uW"}) + 'input': {"token": "jZAC51wHuWdwvQnxwbP2T55vh6R5R2uW"}, + 'output': {"token": f"jZAC{mask}R2uW"} }, { - 'input': json.dumps({"auth": "q2BTIw85kqmjtXl3zCgSSR26gwCGVFMK"}), - 'output': json.dumps({"auth": f"q2BT{mask}VFMK"}) + 'input': {"auth": "q2BTIw85kqmjtXl3zCgSSR26gwCGVFMK"}, + 'output': {"auth": f"q2BT{mask}VFMK"} }, { - 'input': json.dumps({"sessionid": "p1xqXSf2HhYWa2ml6R5R2uWwbP2T55vh"}), - 'output': json.dumps({"sessionid": f"p1xq{mask}55vh"}) + 'input': {"sessionid": "p1xqXSf2HhYWa2ml6R5R2uWwbP2T55vh"}, + 'output': {"sessionid": f"p1xq{mask}55vh"} }, { - 'input': json.dumps({"password": "HlphkcKgQKvofQHP"}), - 'output': json.dumps({"password": mask}) + 'input': {"password": "HlphkcKgQKvofQHP"}, + 'output': {"password": mask} + }, + { + 'input': {"result": "p1xqXSf2HhYWa2ml6R5R2uWwbP2T55vh"}, + 'output': {"result": f"p1xq{mask}55vh"} + }, + { + 'input': {"result": "6.0.0"}, + 'output': {"result": "6.0.0"} } ] diff --git a/zabbix_utils/api.py b/zabbix_utils/api.py index 7865439..a20a3f8 100644 --- a/zabbix_utils/api.py +++ b/zabbix_utils/api.py @@ -425,9 +425,9 @@ def send_api_request(self, method: str, params: Union[dict, None] = None, headers["Authorization"] = f"Basic {self.__basic_cred}" log.debug( - "Sending request to %s with body:%s", + "Sending request to %s with body: %s", self.url, - json.dumps(request_json) + request_json ) req = ul.Request( @@ -457,20 +457,20 @@ def send_api_request(self, method: str, params: Union[dict, None] = None, if method not in ModuleUtils.FILES_METHODS: log.debug( "Received response body: %s", - json.dumps(resp_json, indent=4, separators=(',', ': ')) + resp_json ) else: debug_json = resp_json.copy() if debug_json.get('result'): debug_json['result'] = shorten(debug_json['result'], 200, placeholder='...') log.debug( - "Received response body (short): %s", + "Received response body (clipped): %s", json.dumps(debug_json, indent=4, separators=(',', ': ')) ) if 'error' in resp_json: err = resp_json['error'].copy() - err['body'] = json.dumps(request_json) + err['body'] = request_json.copy() raise APIRequestError(err) return resp_json diff --git a/zabbix_utils/common.py b/zabbix_utils/common.py index 9d8255c..7fd0fd5 100644 --- a/zabbix_utils/common.py +++ b/zabbix_utils/common.py @@ -48,11 +48,11 @@ class ModuleUtils(): # List of private fields and regular expressions to hide them PRIVATE_FIELDS = { - "token": "[A-Za-z0-9]+", - "auth": "[A-Za-z0-9]+", - "sessionid": "[A-Za-z0-9]+", - "password": "[^'\"]+", - "result": "(?!(zabbix_export|[0-9.]{5}))[A-Za-z0-9]+", + "token": r"^.+$", + "auth": r"^.+$", + "sessionid": r"^.+$", + "password": r"^.+$", + "result": r"^(?!(zabbix_export|[0-9.]{5}))[A-Za-z0-9]+$", } @classmethod @@ -96,27 +96,36 @@ def mask_secret(cls, string: str, show_len: int = 4) -> str: return f"{string[:show_len]}{cls.HIDING_MASK}{string[-show_len:]}" @classmethod - def hide_private(cls, message: str, fields: dict = None) -> str: + def hide_private(cls, input_data: dict, fields: dict = None) -> dict: """Hide private data Zabbix info (e.g. token, password) Args: - message (str): Message text with private data. - fields (dict): Dictionary of private fields and their seeking regexps. + input_data (dict): Input dictionary with private fields. + fields (dict): Dictionary of private fields and their filtering regexps. Returns: - str: Message text without private data. + dict: Result dictionary without private data. """ private_fields = fields if fields else cls.PRIVATE_FIELDS + if not isinstance(input_data, dict): + raise TypeError(f"Unsupported data type '{type(input_data).__name__}', \ +only 'dict' is expected") + def gen_repl(match: Match): return cls.mask_secret(match.group(0)) - pattern = re.compile( - r"|".join([rf"(?<=\"{f}\":\s\"){r}" for f, r in private_fields.items()]) - ) + result_data = input_data.copy() + + for key, value in result_data.items(): + if isinstance(value, str): + if key in private_fields: + result_data[key] = re.sub(private_fields[key], gen_repl, value) + if isinstance(value, dict): + result_data[key] = cls.hide_private(value) - return re.sub(pattern, gen_repl, message) + return result_data class ZabbixProtocol(): diff --git a/zabbix_utils/exceptions.py b/zabbix_utils/exceptions.py index 49c4870..58e2e05 100644 --- a/zabbix_utils/exceptions.py +++ b/zabbix_utils/exceptions.py @@ -22,6 +22,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from typing import Union + from .common import ModuleUtils @@ -33,13 +35,12 @@ class APIRequestError(ModuleBaseException): """Exception class when Zabbix API returns error by request. Args: - api_error (str): Raw error message from Zabbix API. + api_error (Union[str, dict]): Raw error message from Zabbix API. """ - def __init__(self, api_error: str): + def __init__(self, api_error: Union[str, dict]): if isinstance(api_error, dict): api_error['body'] = ModuleUtils.hide_private(api_error['body']) super().__init__("{message} {data}".format(**api_error)) - self.error = api_error for key, value in api_error.items(): setattr(self, key, value) else: diff --git a/zabbix_utils/logger.py b/zabbix_utils/logger.py index 72fcf40..34960a2 100644 --- a/zabbix_utils/logger.py +++ b/zabbix_utils/logger.py @@ -22,6 +22,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +import json import logging from .common import ModuleUtils @@ -38,12 +39,15 @@ class SensitiveFilter(logging.Filter): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.hide_data = ModuleUtils.hide_private + + def __hide_data(self, raw_data): + return json.dumps(ModuleUtils.hide_private(raw_data), indent=4, separators=(',', ': ')) def filter(self, record): - record.msg = self.hide_data(record.msg) - if record.args: - record.args = tuple(self.hide_data(arg) if isinstance(arg, str) - else arg for arg in record.args) + if isinstance(record.args, tuple): + record.args = tuple(self.__hide_data(arg) + if isinstance(arg, dict) else arg for arg in record.args) + if isinstance(record.args, dict): + record.args = self.__hide_data(record.args) return 1 From cfa6851a3be144e14944302cf2be69b7f15fd8d2 Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Wed, 27 Dec 2023 11:14:28 +0200 Subject: [PATCH 4/8] updated setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 98dc87e..27cd3bc 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ packages=["zabbix_utils"], tests_require=["unittest"], install_requires=["typing_extensions>=4.0.0;python_version<'3.11'"], - python_requires='>=3.7', + python_requires='>=3.8', project_urls={ 'Zabbix': 'https://www.zabbix.com/documentation/current', 'Source': 'https://github.com/zabbix/python-zabbix-utils', From 10f49c3182db8c6085dfbc710453df5aedf5b23d Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Wed, 27 Dec 2023 11:16:26 +0200 Subject: [PATCH 5/8] tested on Python 3.12 --- .github/workflows/tests.yaml | 1 + README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 09d1977..9bed6bc 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -18,6 +18,7 @@ jobs: - "3.9" - "3.10" - "3.11" + - "3.12" platform: - ubuntu-latest - macos-latest diff --git a/README.md b/README.md index 19f13f8..2f08fc0 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Supported versions: Tested on: * Zabbix 5.0, 6.0, 6.4 and pre-7.0 -* Python 3.8, 3.9, 3.10 and 3.11 +* Python 3.8, 3.9, 3.10, 3.11 and 3.12 ## Documentation From 1465e6d77dcf460a00eed16bb588ca7d3921ba92 Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Mon, 8 Jan 2024 13:48:19 +0200 Subject: [PATCH 6/8] fixed issue with hiding private (sensitive) information --- tests/test_zabbix_api.py | 20 ++++++++++++++++++++ zabbix_utils/common.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_zabbix_api.py b/tests/test_zabbix_api.py index ad03a33..b00f24f 100644 --- a/tests/test_zabbix_api.py +++ b/tests/test_zabbix_api.py @@ -504,6 +504,26 @@ def test_hide_private(self): { 'input': {"result": "6.0.0"}, 'output': {"result": "6.0.0"} + }, + { + 'input': {"result": ["10"]}, + 'output': {"result": ["10"]} + }, + { + 'input': {"result": [{"token": "jZAC51wHuWdwvQnxwbP2T55vh6R5R2uW"}]}, + 'output': {"result": [{"token": f"jZAC{mask}R2uW"}]} + }, + { + 'input': {"result": [["10"],["15"]]}, + 'output': {"result": [["10"],["15"]]} + }, + { + 'input': {"result": [[{"token": "jZAC51wHuWdwvQnxwbP2T55vh6R5R2uW"}]]}, + 'output': {"result": [[{"token": f"jZAC{mask}R2uW"}]]} + }, + { + 'input': {"result": ["jZAC51wHuWdwvQnxwbP2T55vh6R5R2uW"]}, + 'output': {"result": [f"jZAC{mask}R2uW"]} } ] diff --git a/zabbix_utils/common.py b/zabbix_utils/common.py index 7fd0fd5..b8a4fe9 100644 --- a/zabbix_utils/common.py +++ b/zabbix_utils/common.py @@ -52,7 +52,7 @@ class ModuleUtils(): "auth": r"^.+$", "sessionid": r"^.+$", "password": r"^.+$", - "result": r"^(?!(zabbix_export|[0-9.]{5}))[A-Za-z0-9]+$", + "result": r"^[A-Za-z0-9]{32}$", } @classmethod @@ -116,14 +116,38 @@ def hide_private(cls, input_data: dict, fields: dict = None) -> dict: def gen_repl(match: Match): return cls.mask_secret(match.group(0)) + def hide_str(k, v): + return re.sub(private_fields[k], gen_repl, v) + + def hide_dict(v): + return cls.hide_private(v) + + def hide_list(v): + result = [] + for item in v: + if isinstance(item, dict): + result.append(hide_dict(item)) + continue + if isinstance(item, list): + result.append(hide_list(item)) + continue + if isinstance(item, str): + if 'result' in private_fields: + result.append(hide_str('result', item)) + continue + result.append(item) + return result + result_data = input_data.copy() for key, value in result_data.items(): if isinstance(value, str): if key in private_fields: - result_data[key] = re.sub(private_fields[key], gen_repl, value) + result_data[key] = hide_str(key, value) if isinstance(value, dict): - result_data[key] = cls.hide_private(value) + result_data[key] = hide_dict(value) + if isinstance(value, list): + result_data[key] = hide_list(value) return result_data From c5e31d55174feb8cbdfa7d5839d64c17c5106c7b Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Tue, 9 Jan 2024 11:19:57 +0200 Subject: [PATCH 7/8] updated tests workflow --- .github/workflows/tests.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9bed6bc..19fbfb8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,6 @@ jobs: strategy: matrix: python-version: - - "3.7" - "3.8" - "3.9" - "3.10" @@ -32,15 +31,15 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r ./requirements.txt - pip install -r ./requirements-dev.txt - pip install coverage + python -m pip install --upgrade pip + pip install -r ./requirements.txt + pip install -r ./requirements-dev.txt + pip install coverage - name: Lint with flake8 run: | flake8 ./zabbix_utils/ --count --select=E9,F63,F7,F82 --show-source --statistics flake8 ./zabbix_utils/ --count --exit-zero --max-complexity=20 --max-line-length=127 --statistics - name: Test with unittest run: | - python -m unittest discover -s ./tests -p 'test_*.py' + python -m unittest discover -s ./tests -p 'test_*.py' From 1b2e268d6d1a6964f209509691a10c3218eaea24 Mon Sep 17 00:00:00 2001 From: Aleksandr Iantsen Date: Tue, 9 Jan 2024 11:30:35 +0200 Subject: [PATCH 8/8] release v1.0.3 --- CHANGELOG.md | 12 ++++++++++++ zabbix_utils/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79a1bae..6effd00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## [1.0.3](https://github.com/zabbix/python-zabbix-utils/compare/v1.0.2...v1.0.3) (2024-01-09) + +### Documentation + +- added support for Python 3.12 +- discontinued support for Python 3.7 + +### Bug fixes: + +- fixed issue with hiding private (sensitive) information in the log. +- fixed small bugs and flaws. + ## [1.0.2](https://github.com/zabbix/python-zabbix-utils/compare/v1.0.1...v1.0.2) (2023-12-15) ### Bug fixes: diff --git a/zabbix_utils/version.py b/zabbix_utils/version.py index 2179325..e0e88d9 100644 --- a/zabbix_utils/version.py +++ b/zabbix_utils/version.py @@ -22,7 +22,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -__version__ = "1.0.2" +__version__ = "1.0.3" __min_supported__ = 5.0 __max_supported__ = 7.0