From 5a85b8f7b6912de63bacc4759fc04ebad1fbfae0 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 19 Oct 2023 11:47:55 -0500 Subject: [PATCH 01/33] config manager class --- az-config.yml | 4 + .../utils/config_manager.py | 73 +++++++++++++++++++ test.py | 3 + 3 files changed, 80 insertions(+) create mode 100644 az-config.yml create mode 100644 azure_functions_worker/utils/config_manager.py create mode 100644 test.py diff --git a/az-config.yml b/az-config.yml new file mode 100644 index 000000000..c05ae579f --- /dev/null +++ b/az-config.yml @@ -0,0 +1,4 @@ +SCRIPT_FILE_NAME: test.py +SCRIPT_FILE_NAME2: test2.py +SCRIPT_FILE_NAME3: test3.py +SCRIPT_FILE_NAME4: test4.py \ No newline at end of file diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py new file mode 100644 index 000000000..87dae80c5 --- /dev/null +++ b/azure_functions_worker/utils/config_manager.py @@ -0,0 +1,73 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import yaml +from typing import Optional, Callable + +config_data = {} + +def read_config() -> dict: + with open("az-config.yml", "r") as stream: + try: + config_data = yaml.safe_load(stream) + config_data = dict((k.lower(), v) for k,v in config_data.items()) + except yaml.YAMLError as exc: + print(exc) + return config_data + + +def write_config(config_data: dict): + with open("az-config.yml", "w") as stream: + try: + yaml.dump(config_data, stream) + except yaml.YAMLError as exc: + print(exc) + + +def is_true_like(setting: str) -> bool: + if setting is None: + return False + + return setting.lower().strip() in {'1', 'true', 't', 'yes', 'y'} + + +def is_false_like(setting: str) -> bool: + if setting is None: + return False + + return setting.lower().strip() in {'0', 'false', 'f', 'no', 'n'} + + +def is_envvar_true(key: str) -> bool: + if config_data.get(key.lower) is None: + return False + + return is_true_like(config_data.get(key.lower)) + + +def is_envvar_false(key: str) -> bool: + if config_data.get(key.lower) is None: + return False + + return is_false_like(config_data.get(key.lower)) + + +def get_env_var( + setting: str, + default_value: Optional[str] = None, + validator: Optional[Callable[[str], bool]] = None +) -> Optional[str]: + app_setting_value = config_data.get(setting.lower) + + if app_setting_value is None: + return default_value + + if validator is None: + return app_setting_value + + if validator(app_setting_value): + return app_setting_value + return default_value + + +def set_env_var(setting: str, value: str): + config_data.update({setting.lower: value}) \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 000000000..9fe7c73a0 --- /dev/null +++ b/test.py @@ -0,0 +1,3 @@ +from azure_functions_worker.utils.config_manager import read_config + +print(read_config()) From 23e2ddabe10567ff2410193f9372a4e96543b1b9 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 18 Jan 2024 10:25:07 -0600 Subject: [PATCH 02/33] basic working prototype --- az-config.yml | 4 ---- azure_functions_worker/dispatcher.py | 2 ++ azure_functions_worker/utils/common.py | 3 +++ .../utils/config_manager.py | 19 +++++++++++-------- test.py | 3 --- 5 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 az-config.yml delete mode 100644 test.py diff --git a/az-config.yml b/az-config.yml deleted file mode 100644 index c05ae579f..000000000 --- a/az-config.yml +++ /dev/null @@ -1,4 +0,0 @@ -SCRIPT_FILE_NAME: test.py -SCRIPT_FILE_NAME2: test2.py -SCRIPT_FILE_NAME3: test3.py -SCRIPT_FILE_NAME4: test4.py \ No newline at end of file diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 66f89e219..003716f81 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -32,6 +32,7 @@ from .logging import (logger, error_logger, is_system_log_category, CONSOLE_LOG_PREFIX, format_exception) from .utils.common import get_app_setting, is_envvar_true +from .utils.config_manager import read_config from .utils.dependency import DependencyManager from .utils.tracing import marshall_exception_trace from .utils.wrappers import disable_feature_by @@ -286,6 +287,7 @@ async def _handle__worker_init_request(self, request): constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } + read_config(os.path.join(worker_init_request.function_app_directory,"az-config.yml")) # Can detech worker packages only when customer's code is present # This only works in dedicated and premium sku. # The consumption sku will switch on environment_reload request. diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index 0f12a5f5c..afe78e4dd 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -5,6 +5,7 @@ import sys from types import ModuleType from typing import Optional, Callable +import azure_functions_worker.utils.config_manager as config_manager from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH, \ PYTHON_EXTENSIONS_RELOAD_FUNCTIONS @@ -25,6 +26,8 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(env_key: str) -> bool: + if config_manager.config_exists() and config_manager.is_envvar_true(env_key): + return True if os.getenv(env_key) is None: return False diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index 87dae80c5..ec7afac9b 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -5,11 +5,12 @@ config_data = {} -def read_config() -> dict: - with open("az-config.yml", "r") as stream: +def read_config(function_path: str) -> dict: + with open(function_path, "r") as stream: try: + global config_data config_data = yaml.safe_load(stream) - config_data = dict((k.lower(), v) for k,v in config_data.items()) + config_data = dict((k, v) for k,v in config_data.items()) except yaml.YAMLError as exc: print(exc) return config_data @@ -22,6 +23,8 @@ def write_config(config_data: dict): except yaml.YAMLError as exc: print(exc) +def config_exists() -> bool: + return config_data is not {} def is_true_like(setting: str) -> bool: if setting is None: @@ -38,17 +41,17 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: - if config_data.get(key.lower) is None: + if config_data.get(key) is None: return False - return is_true_like(config_data.get(key.lower)) + return is_true_like(config_data.get(key)) def is_envvar_false(key: str) -> bool: - if config_data.get(key.lower) is None: + if config_data.get(key) is None: return False - return is_false_like(config_data.get(key.lower)) + return is_false_like(config_data.get(key)) def get_env_var( @@ -56,7 +59,7 @@ def get_env_var( default_value: Optional[str] = None, validator: Optional[Callable[[str], bool]] = None ) -> Optional[str]: - app_setting_value = config_data.get(setting.lower) + app_setting_value = config_data.get(setting) if app_setting_value is None: return default_value diff --git a/test.py b/test.py deleted file mode 100644 index 9fe7c73a0..000000000 --- a/test.py +++ /dev/null @@ -1,3 +0,0 @@ -from azure_functions_worker.utils.config_manager import read_config - -print(read_config()) From 078a305478937e65cf6ea9616692bf6af9acb7ee Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 18 Jan 2024 10:33:04 -0600 Subject: [PATCH 03/33] basic fully working prototype --- azure_functions_worker/utils/common.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index afe78e4dd..a36c81290 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -35,6 +35,8 @@ def is_envvar_true(env_key: str) -> bool: def is_envvar_false(env_key: str) -> bool: + if config_manager.config_exists() and config_manager.is_envvar_false(env_key): + return False if os.getenv(env_key) is None: return False @@ -71,6 +73,16 @@ def get_app_setting( Optional[str] A string value that is set in the application setting """ + value_to_return = default_value + + # Check first if setting is in config file + if config_manager.config_exists(): + value_to_return = config_manager.get_env_var(setting, value_to_return, validator) + # If the value is not the default value, we should return it -- has been set in the config file + if value_to_return is not default_value: + return value_to_return + + # Now checking from env variables app_setting_value = os.getenv(setting) # If an app setting is not configured, we return the default value From bb70497bd9109363a20470bfe6de704841f6ef0f Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Thu, 18 Jan 2024 13:57:05 -0600 Subject: [PATCH 04/33] refactor common to config_manager --- .../file_accessor_factory.py | 2 +- .../file_accessor_unix.py | 2 +- .../shared_memory_manager.py | 2 +- azure_functions_worker/dispatcher.py | 6 +- azure_functions_worker/utils/common.py | 89 +------------------ .../utils/config_manager.py | 88 +++++++++++++----- azure_functions_worker/utils/dependency.py | 2 +- azure_functions_worker/utils/wrappers.py | 2 +- .../test_dependency_isolation_functions.py | 2 +- tests/endtoend/test_durable_functions.py | 2 +- .../endtoend/test_eventhub_batch_functions.py | 2 +- tests/endtoend/test_table_functions.py | 2 +- tests/unittests/test_shared_memory_manager.py | 2 +- tests/unittests/test_utilities.py | 59 ++++++------ tests/utils/testutils.py | 3 +- 15 files changed, 115 insertions(+), 150 deletions(-) diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py index 1c0340222..f6ed531a1 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py @@ -8,7 +8,7 @@ from .file_accessor_unix import FileAccessorUnix from .file_accessor_windows import FileAccessorWindows from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED -from ...utils.common import is_envvar_true +from ...utils.config_manager import is_envvar_true class FileAccessorFactory: diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py index aaf122108..40f6d9118 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py @@ -9,7 +9,7 @@ from .shared_memory_constants import SharedMemoryConstants as consts from .shared_memory_exception import SharedMemoryException from .file_accessor import FileAccessor -from ...utils.common import get_app_setting +from ...utils.config_manager import get_app_setting from ...logging import logger diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py index a74027b7a..05d2f8c2f 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py @@ -9,7 +9,7 @@ from .shared_memory_map import SharedMemoryMap from ..datumdef import Datum from ...logging import logger -from ...utils.common import is_envvar_true +from ...utils.config_manager import is_envvar_true from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 003716f81..cb9d4d87c 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -31,8 +31,7 @@ from .logging import disable_console_logging, enable_console_logging from .logging import (logger, error_logger, is_system_log_category, CONSOLE_LOG_PREFIX, format_exception) -from .utils.common import get_app_setting, is_envvar_true -from .utils.config_manager import read_config +from .utils.config_manager import read_config, is_envvar_true, get_app_setting from .utils.dependency import DependencyManager from .utils.tracing import marshall_exception_trace from .utils.wrappers import disable_feature_by @@ -287,7 +286,8 @@ async def _handle__worker_init_request(self, request): constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } - read_config(os.path.join(worker_init_request.function_app_directory,"az-config.yml")) + read_config(os.path.join(worker_init_request.function_app_directory, + "az-config.yml")) # Can detech worker packages only when customer's code is present # This only works in dedicated and premium sku. # The consumption sku will switch on environment_reload request. diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index a36c81290..8c55e3c59 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -1,106 +1,19 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import importlib -import os import sys from types import ModuleType -from typing import Optional, Callable import azure_functions_worker.utils.config_manager as config_manager from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH, \ PYTHON_EXTENSIONS_RELOAD_FUNCTIONS -def is_true_like(setting: str) -> bool: - if setting is None: - return False - - return setting.lower().strip() in {'1', 'true', 't', 'yes', 'y'} - - -def is_false_like(setting: str) -> bool: - if setting is None: - return False - - return setting.lower().strip() in {'0', 'false', 'f', 'no', 'n'} - - -def is_envvar_true(env_key: str) -> bool: - if config_manager.config_exists() and config_manager.is_envvar_true(env_key): - return True - if os.getenv(env_key) is None: - return False - - return is_true_like(os.environ[env_key]) - - -def is_envvar_false(env_key: str) -> bool: - if config_manager.config_exists() and config_manager.is_envvar_false(env_key): - return False - if os.getenv(env_key) is None: - return False - - return is_false_like(os.environ[env_key]) - - def is_python_version(version: str) -> bool: current_version = f'{sys.version_info.major}.{sys.version_info.minor}' return current_version == version -def get_app_setting( - setting: str, - default_value: Optional[str] = None, - validator: Optional[Callable[[str], bool]] = None -) -> Optional[str]: - """Returns the application setting from environment variable. - - Parameters - ---------- - setting: str - The name of the application setting (e.g. FUNCTIONS_RUNTIME_VERSION) - - default_value: Optional[str] - The expected return value when the application setting is not found, - or the app setting does not pass the validator. - - validator: Optional[Callable[[str], bool]] - A function accepts the app setting value and should return True when - the app setting value is acceptable. - - Returns - ------- - Optional[str] - A string value that is set in the application setting - """ - value_to_return = default_value - - # Check first if setting is in config file - if config_manager.config_exists(): - value_to_return = config_manager.get_env_var(setting, value_to_return, validator) - # If the value is not the default value, we should return it -- has been set in the config file - if value_to_return is not default_value: - return value_to_return - - # Now checking from env variables - app_setting_value = os.getenv(setting) - - # If an app setting is not configured, we return the default value - if app_setting_value is None: - return default_value - - # If there's no validator, we should return the app setting value directly - if validator is None: - return app_setting_value - - # If the app setting is set with a validator, - # On True, should return the app setting value - # On False, should return the default value - if validator(app_setting_value): - return app_setting_value - return default_value - - def get_sdk_version(module: ModuleType) -> str: """Check the version of azure.functions sdk. @@ -129,7 +42,7 @@ def get_sdk_from_sys_path() -> ModuleType: The azure.functions that is loaded from the first sys.path entry """ - if is_envvar_true(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS): + if config_manager.is_envvar_true(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS): backup_azure_functions = None backup_azure = None diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index ec7afac9b..ee8696e32 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -1,19 +1,20 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os import yaml from typing import Optional, Callable config_data = {} -def read_config(function_path: str) -> dict: + +def read_config(function_path: str): with open(function_path, "r") as stream: try: global config_data config_data = yaml.safe_load(stream) - config_data = dict((k, v) for k,v in config_data.items()) + config_data = dict((k, v) for k, v in config_data.items()) except yaml.YAMLError as exc: print(exc) - return config_data def write_config(config_data: dict): @@ -23,9 +24,11 @@ def write_config(config_data: dict): except yaml.YAMLError as exc: print(exc) + def config_exists() -> bool: return config_data is not {} + def is_true_like(setting: str) -> bool: if setting is None: return False @@ -41,36 +44,81 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: - if config_data.get(key) is None: - return False - - return is_true_like(config_data.get(key)) + # Check first from config file + if config_exists() and config_data.get(key) is not None: + return is_true_like(config_data.get(key)) + # Now check from environment variables + elif os.getenv(key) is not None: + return is_true_like(os.environ[key]) + # Var not found in config file or environment variables, return False + return False def is_envvar_false(key: str) -> bool: - if config_data.get(key) is None: - return False - - return is_false_like(config_data.get(key)) - - -def get_env_var( - setting: str, + # Check first from config file + if config_exists() and config_data.get(key) is not None: + return is_false_like(config_data.get(key)) + # Now check from environment variables + elif os.getenv(key) is not None: + return is_false_like(os.environ[key]) + # Var not found in config file or environment variables, return False + return False + + +def get_app_setting( + setting: str, default_value: Optional[str] = None, validator: Optional[Callable[[str], bool]] = None ) -> Optional[str]: - app_setting_value = config_data.get(setting) - + """Returns the application setting from environment variable. + + Parameters + ---------- + setting: str + The name of the application setting (e.g. FUNCTIONS_RUNTIME_VERSION) + + default_value: Optional[str] + The expected return value when the application setting is not found, + or the app setting does not pass the validator. + + validator: Optional[Callable[[str], bool]] + A function accepts the app setting value and should return True when + the app setting value is acceptable. + + Returns + ------- + Optional[str] + A string value that is set in the application setting + """ + # Check first from config file + if config_exists() and config_data.get(setting) is not None: + # Setting exists, check with validator + app_setting_value = config_data.get(setting) + if validator is None: + return app_setting_value + else: + if validator(app_setting_value): + return app_setting_value + return default_value + + # Setting not in config file, now check from environment variables + app_setting_value = os.getenv(setting) + + # If an app setting is not configured, we return the default value if app_setting_value is None: return default_value - + + # If there's no validator, we should return the app setting value directly if validator is None: return app_setting_value - + + # If the app setting is set with a validator, + # On True, should return the app setting value + # On False, should return the default value if validator(app_setting_value): return app_setting_value return default_value def set_env_var(setting: str, value: str): - config_data.update({setting.lower: value}) \ No newline at end of file + config_data.update({setting.lower: value}) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index a213b5e8e..d3a06bdbf 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -8,7 +8,7 @@ from types import ModuleType from typing import List, Optional -from azure_functions_worker.utils.common import is_true_like +from azure_functions_worker.utils.config_manager import is_true_like from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, CONTAINER_NAME, diff --git a/azure_functions_worker/utils/wrappers.py b/azure_functions_worker/utils/wrappers.py index 2a4afd64c..d306888e1 100644 --- a/azure_functions_worker/utils/wrappers.py +++ b/azure_functions_worker/utils/wrappers.py @@ -3,7 +3,7 @@ from typing import Callable, Any -from .common import is_envvar_true, is_envvar_false +from .config_manager import is_envvar_true, is_envvar_false from .tracing import extend_exception_message from ..logging import logger diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index c2f5cedb8..457d63ec2 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -8,7 +8,7 @@ from requests import Response -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true from tests.utils import testutils from tests.utils.constants import PYAZURE_INTEGRATION_TEST, \ CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST diff --git a/tests/endtoend/test_durable_functions.py b/tests/endtoend/test_durable_functions.py index 4bb5ed433..fd83c29ee 100644 --- a/tests/endtoend/test_durable_functions.py +++ b/tests/endtoend/test_durable_functions.py @@ -8,7 +8,7 @@ import requests -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true from tests.utils import testutils from tests.utils.constants import DEDICATED_DOCKER_TEST, CONSUMPTION_DOCKER_TEST diff --git a/tests/endtoend/test_eventhub_batch_functions.py b/tests/endtoend/test_eventhub_batch_functions.py index da12824e2..aaa0cf14a 100644 --- a/tests/endtoend/test_eventhub_batch_functions.py +++ b/tests/endtoend/test_eventhub_batch_functions.py @@ -8,7 +8,7 @@ from dateutil import parser, tz -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true from tests.utils import testutils from tests.utils.constants import DEDICATED_DOCKER_TEST, CONSUMPTION_DOCKER_TEST diff --git a/tests/endtoend/test_table_functions.py b/tests/endtoend/test_table_functions.py index c9290e1ea..7068ab0ea 100644 --- a/tests/endtoend/test_table_functions.py +++ b/tests/endtoend/test_table_functions.py @@ -5,7 +5,7 @@ import time from unittest import skipIf -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true from tests.utils import testutils from tests.utils.constants import DEDICATED_DOCKER_TEST, \ CONSUMPTION_DOCKER_TEST diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index 60a89e3ea..b5d582945 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -7,7 +7,7 @@ import sys from unittest import skipIf from unittest.mock import patch -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true from azure.functions import meta as bind_meta from tests.utils import testutils from azure_functions_worker.bindings.shared_memory_data_transfer \ diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index c71bcce37..23328557b 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -8,7 +8,7 @@ from unittest.mock import patch from azure_functions_worker.constants import PYTHON_EXTENSIONS_RELOAD_FUNCTIONS -from azure_functions_worker.utils import common, wrappers +from azure_functions_worker.utils import common, config_manager, wrappers TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" TEST_FEATURE_FLAG = "APP_SETTING_FEATURE_FLAG" @@ -89,44 +89,44 @@ def tearDown(self): self.mock_environ.stop() def test_is_true_like_accepted(self): - self.assertTrue(common.is_true_like('1')) - self.assertTrue(common.is_true_like('true')) - self.assertTrue(common.is_true_like('T')) - self.assertTrue(common.is_true_like('YES')) - self.assertTrue(common.is_true_like('y')) + self.assertTrue(config_manager.is_true_like('1')) + self.assertTrue(config_manager.is_true_like('true')) + self.assertTrue(config_manager.is_true_like('T')) + self.assertTrue(config_manager.is_true_like('YES')) + self.assertTrue(config_manager.is_true_like('y')) def test_is_true_like_rejected(self): - self.assertFalse(common.is_true_like(None)) - self.assertFalse(common.is_true_like('')) - self.assertFalse(common.is_true_like('secret')) + self.assertFalse(config_manager.is_true_like(None)) + self.assertFalse(config_manager.is_true_like('')) + self.assertFalse(config_manager.is_true_like('secret')) def test_is_false_like_accepted(self): - self.assertTrue(common.is_false_like('0')) - self.assertTrue(common.is_false_like('false')) - self.assertTrue(common.is_false_like('F')) - self.assertTrue(common.is_false_like('NO')) - self.assertTrue(common.is_false_like('n')) + self.assertTrue(config_manager.is_false_like('0')) + self.assertTrue(config_manager.is_false_like('false')) + self.assertTrue(config_manager.is_false_like('F')) + self.assertTrue(config_manager.is_false_like('NO')) + self.assertTrue(config_manager.is_false_like('n')) def test_is_false_like_rejected(self): - self.assertFalse(common.is_false_like(None)) - self.assertFalse(common.is_false_like('')) - self.assertFalse(common.is_false_like('secret')) + self.assertFalse(config_manager.is_false_like(None)) + self.assertFalse(config_manager.is_false_like('')) + self.assertFalse(config_manager.is_false_like('secret')) def test_is_envvar_true(self): os.environ[TEST_FEATURE_FLAG] = 'true' - self.assertTrue(common.is_envvar_true(TEST_FEATURE_FLAG)) + self.assertTrue(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_is_envvar_not_true_on_unset(self): self._unset_feature_flag() - self.assertFalse(common.is_envvar_true(TEST_FEATURE_FLAG)) + self.assertFalse(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_is_envvar_false(self): os.environ[TEST_FEATURE_FLAG] = 'false' - self.assertTrue(common.is_envvar_false(TEST_FEATURE_FLAG)) + self.assertTrue(config_manager.is_envvar_false(TEST_FEATURE_FLAG)) def test_is_envvar_not_false_on_unset(self): self._unset_feature_flag() - self.assertFalse(common.is_envvar_true(TEST_FEATURE_FLAG)) + self.assertFalse(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_disable_feature_with_no_feature_flag(self): mock_feature = MockFeature() @@ -244,7 +244,7 @@ def test_exception_message_should_not_be_extended_on_other_exception(self): self.assertEqual(type(e), ValueError) def test_app_settings_not_set_should_return_none(self): - app_setting = common.get_app_setting(TEST_APP_SETTING_NAME) + app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME) self.assertIsNone(app_setting) def test_app_settings_should_return_value(self): @@ -252,11 +252,12 @@ def test_app_settings_should_return_value(self): os.environ.update({TEST_APP_SETTING_NAME: '42'}) # Try using utility to acquire application setting - app_setting = common.get_app_setting(TEST_APP_SETTING_NAME) + app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME) self.assertEqual(app_setting, '42') def test_app_settings_not_set_should_return_default_value(self): - app_setting = common.get_app_setting(TEST_APP_SETTING_NAME, 'default') + app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, + 'default') self.assertEqual(app_setting, 'default') def test_app_settings_should_ignore_default_value(self): @@ -264,14 +265,16 @@ def test_app_settings_should_ignore_default_value(self): os.environ.update({TEST_APP_SETTING_NAME: '42'}) # Try using utility to acquire application setting - app_setting = common.get_app_setting(TEST_APP_SETTING_NAME, 'default') + app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, + 'default') self.assertEqual(app_setting, '42') def test_app_settings_should_not_trigger_validator_when_not_set(self): def raise_excpt(value: str): raise Exception('Should not raise on app setting not found') - common.get_app_setting(TEST_APP_SETTING_NAME, validator=raise_excpt) + config_manager.get_app_setting(TEST_APP_SETTING_NAME, + validator=raise_excpt) def test_app_settings_return_default_value_when_validation_fail(self): def parse_int_no_raise(value: str): @@ -284,7 +287,7 @@ def parse_int_no_raise(value: str): # Set application setting to an invalid value os.environ.update({TEST_APP_SETTING_NAME: 'invalid'}) - app_setting = common.get_app_setting( + app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, default_value='1', validator=parse_int_no_raise @@ -304,7 +307,7 @@ def parse_int_no_raise(value: str): # Set application setting to an invalid value os.environ.update({TEST_APP_SETTING_NAME: '42'}) - app_setting = common.get_app_setting( + app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, default_value='1', validator=parse_int_no_raise diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index 6f6fb76af..229601682 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -45,7 +45,8 @@ FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, UNIX_SHARED_MEMORY_DIRECTORIES ) -from azure_functions_worker.utils.common import is_envvar_true, get_app_setting +from azure_functions_worker.utils.config_manager import (is_envvar_true, + get_app_setting) from tests.utils.constants import PYAZURE_WORKER_DIR, \ PYAZURE_INTEGRATION_TEST, PROJECT_ROOT, WORKER_CONFIG, \ CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST, PYAZURE_WEBHOST_DEBUG From a3e0af17b90084b5a83cefca984a5cb534b6a086 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 6 Feb 2024 14:34:42 -0600 Subject: [PATCH 05/33] combine os env and config file --- azure_functions_worker/dispatcher.py | 3 ++ .../utils/config_manager.py | 52 +++++++------------ azure_functions_worker/utils/dependency.py | 12 +++-- 3 files changed, 29 insertions(+), 38 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index cb9d4d87c..cab8b873e 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -569,6 +569,9 @@ async def _handle__function_environment_reload_request(self, request): env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] + read_config(os.path.join( + func_env_reload_request.function_app_directory, + "az-config.yml")) # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index ee8696e32..68cc07afc 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -11,11 +11,18 @@ def read_config(function_path: str): with open(function_path, "r") as stream: try: global config_data + # loads the entire yaml file config_data = yaml.safe_load(stream) - config_data = dict((k, v) for k, v in config_data.items()) + # gets the python section of the yaml file + config_data = config_data.get("PYTHON") except yaml.YAMLError as exc: print(exc) + env_copy = os.environ + # updates the config dictionary with the environment variables + # this prioritizes set env variables over the config file + config_data.update(env_copy) + def write_config(config_data: dict): with open("az-config.yml", "w") as stream: @@ -44,24 +51,14 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: - # Check first from config file if config_exists() and config_data.get(key) is not None: return is_true_like(config_data.get(key)) - # Now check from environment variables - elif os.getenv(key) is not None: - return is_true_like(os.environ[key]) - # Var not found in config file or environment variables, return False return False def is_envvar_false(key: str) -> bool: - # Check first from config file if config_exists() and config_data.get(key) is not None: return is_false_like(config_data.get(key)) - # Now check from environment variables - elif os.getenv(key) is not None: - return is_false_like(os.environ[key]) - # Var not found in config file or environment variables, return False return False @@ -90,33 +87,22 @@ def get_app_setting( Optional[str] A string value that is set in the application setting """ - # Check first from config file if config_exists() and config_data.get(setting) is not None: # Setting exists, check with validator app_setting_value = config_data.get(setting) + + # If there's no validator, return the app setting value directly if validator is None: return app_setting_value - else: - if validator(app_setting_value): - return app_setting_value - return default_value - - # Setting not in config file, now check from environment variables - app_setting_value = os.getenv(setting) - - # If an app setting is not configured, we return the default value - if app_setting_value is None: - return default_value - - # If there's no validator, we should return the app setting value directly - if validator is None: - return app_setting_value - - # If the app setting is set with a validator, - # On True, should return the app setting value - # On False, should return the default value - if validator(app_setting_value): - return app_setting_value + + # If the app setting is set with a validator, + # On True, should return the app setting value + # On False, should return the default value + if validator(app_setting_value): + return app_setting_value + + # Setting is not configured or validator is false + # Return default value return default_value diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index d3a06bdbf..255cf22e6 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -8,7 +8,9 @@ from types import ModuleType from typing import List, Optional -from azure_functions_worker.utils.config_manager import is_true_like +from azure_functions_worker.utils.config_manager import (is_true_like, + is_envvar_true, + get_app_setting) from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, CONTAINER_NAME, @@ -139,7 +141,7 @@ def prioritize_customer_dependencies(cls, cx_working_dir=None): if not working_directory: working_directory = cls.cx_working_dir if not working_directory: - working_directory = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT, '') + working_directory = get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT, '') # Try to get the latest customer's dependency path cx_deps_path: str = cls._get_cx_deps_path() @@ -181,7 +183,7 @@ def reload_customer_libraries(cls, cx_working_dir: str): cx_working_dir: str The path which contains customer's project file (e.g. wwwroot). """ - use_new_env = os.getenv(PYTHON_ISOLATE_WORKER_DEPENDENCIES) + use_new_env = is_envvar_true(PYTHON_ISOLATE_WORKER_DEPENDENCIES) if use_new_env is None: use_new = ( PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if @@ -302,7 +304,7 @@ def _get_cx_deps_path() -> str: Linux Dedicated/Premium: path to customer's site pacakges Linux Consumption: empty string """ - prefix: Optional[str] = os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT) + prefix: Optional[str] = get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT) cx_paths: List[str] = [ p for p in sys.path if prefix and p.startswith(prefix) and ('site-packages' in p) @@ -321,7 +323,7 @@ def _get_cx_working_dir() -> str: Linux Dedicated/Premium: AzureWebJobsScriptRoot env variable Linux Consumption: empty string """ - return os.getenv(AZURE_WEBJOBS_SCRIPT_ROOT, '') + return get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT, '') @staticmethod def _get_worker_deps_path() -> str: From 347b900d536a373b86530dde107c4457c6b3f537 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 6 Feb 2024 15:12:45 -0600 Subject: [PATCH 06/33] yaml dep --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8aa0671c0..da12ba532 100644 --- a/setup.py +++ b/setup.py @@ -144,7 +144,8 @@ "scikit-learn", "opencv-python", "pandas", - "numpy" + "numpy", + "yaml" ] } From 225801b0340d0e3229f496c0c413fe3d8c474934 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 6 Feb 2024 15:37:55 -0600 Subject: [PATCH 07/33] install pyyaml --- .github/workflows/ci_e2e_workflow.yml | 1 + setup.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index 8138121da..c814b561f 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -59,6 +59,7 @@ jobs: python -m pip install --upgrade pip python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U PyYAML --pre python -m pip install -U -e .[dev] # Retry a couple times to avoid certificate issue diff --git a/setup.py b/setup.py index da12ba532..8aa0671c0 100644 --- a/setup.py +++ b/setup.py @@ -144,8 +144,7 @@ "scikit-learn", "opencv-python", "pandas", - "numpy", - "yaml" + "numpy" ] } From 330b06b1bd0cec72536da261a1d7b68b7f233678 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 6 Feb 2024 15:47:57 -0600 Subject: [PATCH 08/33] pyyaml in setup --- .github/workflows/ci_e2e_workflow.yml | 1 - setup.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_e2e_workflow.yml b/.github/workflows/ci_e2e_workflow.yml index c814b561f..8138121da 100644 --- a/.github/workflows/ci_e2e_workflow.yml +++ b/.github/workflows/ci_e2e_workflow.yml @@ -59,7 +59,6 @@ jobs: python -m pip install --upgrade pip python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre - python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U PyYAML --pre python -m pip install -U -e .[dev] # Retry a couple times to avoid certificate issue diff --git a/setup.py b/setup.py index 8aa0671c0..f37a92d2a 100644 --- a/setup.py +++ b/setup.py @@ -144,7 +144,8 @@ "scikit-learn", "opencv-python", "pandas", - "numpy" + "numpy", + "pyyaml" ] } From 9cb58bf7a3cf201a224b5471bb0065a4a311b6c2 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Tue, 20 Feb 2024 11:26:29 -0600 Subject: [PATCH 09/33] read from json --- azure_functions_worker/dispatcher.py | 4 ++-- .../utils/config_manager.py | 20 +++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index cab8b873e..bd62b551d 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -287,7 +287,7 @@ async def _handle__worker_init_request(self, request): } read_config(os.path.join(worker_init_request.function_app_directory, - "az-config.yml")) + "az-config.json")) # Can detech worker packages only when customer's code is present # This only works in dedicated and premium sku. # The consumption sku will switch on environment_reload request. @@ -571,7 +571,7 @@ async def _handle__function_environment_reload_request(self, request): os.environ[var] = env_vars[var] read_config(os.path.join( func_env_reload_request.function_app_directory, - "az-config.yml")) + "az-config.json")) # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index 68cc07afc..675d70e44 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os -import yaml +import json from typing import Optional, Callable config_data = {} @@ -9,14 +9,11 @@ def read_config(function_path: str): with open(function_path, "r") as stream: - try: - global config_data - # loads the entire yaml file - config_data = yaml.safe_load(stream) - # gets the python section of the yaml file - config_data = config_data.get("PYTHON") - except yaml.YAMLError as exc: - print(exc) + global config_data + # loads the entire yaml file + full_config_data = json.load(stream) + # gets the python section of the yaml file + config_data = full_config_data.get("PYTHON") env_copy = os.environ # updates the config dictionary with the environment variables @@ -26,10 +23,7 @@ def read_config(function_path: str): def write_config(config_data: dict): with open("az-config.yml", "w") as stream: - try: - yaml.dump(config_data, stream) - except yaml.YAMLError as exc: - print(exc) + json.dumps(config_data, stream) def config_exists() -> bool: From 409c810e8575b37f656da45b808f739303ff2953 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 28 Feb 2024 16:34:29 -0600 Subject: [PATCH 10/33] some dependency and utilities tests --- .../utils/config_manager.py | 30 +++++----- tests/unittests/test_utilities.py | 43 +++++++++----- tests/unittests/test_utilities_dependency.py | 57 ++++++++++++++----- 3 files changed, 88 insertions(+), 42 deletions(-) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index e61e0dee3..41cdad9c1 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -8,24 +8,21 @@ def read_config(function_path: str): - with open(function_path, "r") as stream: - global config_data - # loads the entire json file - full_config_data = json.load(stream) - # gets the python section of the json file - config_data = full_config_data.get("PYTHON") - + try: + with open(function_path, "r") as stream: + global config_data + # loads the entire json file + full_config_data = json.load(stream) + # gets the python section of the json file + config_data = full_config_data.get("PYTHON") + except FileNotFoundError: + pass env_copy = os.environ # updates the config dictionary with the environment variables # this prioritizes set env variables over the config file config_data.update(env_copy) -def write_config(data: dict): - with open("az-config.json", "w") as stream: - json.dumps(data, stream) - - def config_exists() -> bool: return config_data is not {} @@ -101,4 +98,11 @@ def get_app_setting( def set_env_var(setting: str, value: str): - config_data.update({setting.lower: value}) + global config_data + config_data[setting] = value + test = "" + + +def del_env_var(setting: str): + global config_data + config_data.pop(setting, None) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 6ef4b1fe5..499c1d778 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -113,16 +113,18 @@ def test_is_false_like_rejected(self): self.assertFalse(config_manager.is_false_like('secret')) def test_is_envvar_true(self): - os.environ[TEST_FEATURE_FLAG] = 'true' + config_manager.set_env_var(TEST_FEATURE_FLAG, 'true') self.assertTrue(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) + config_manager.del_env_var(TEST_FEATURE_FLAG) def test_is_envvar_not_true_on_unset(self): self._unset_feature_flag() self.assertFalse(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_is_envvar_false(self): - os.environ[TEST_FEATURE_FLAG] = 'false' + config_manager.set_env_var(TEST_FEATURE_FLAG, 'false') self.assertTrue(config_manager.is_envvar_false(TEST_FEATURE_FLAG)) + config_manager.del_env_var(TEST_FEATURE_FLAG) def test_is_envvar_not_false_on_unset(self): self._unset_feature_flag() @@ -144,12 +146,13 @@ def test_disable_feature_with_default_value(self): def test_enable_feature_with_feature_flag(self): feature_flag = TEST_FEATURE_FLAG - os.environ[feature_flag] = '1' + config_manager.set_env_var(feature_flag, '1') mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_enabled(output) self.assertEqual(result, 'mock_feature_enabled') self.assertListEqual(output, ['mock_feature_enabled']) + config_manager.del_env_var(feature_flag) def test_enable_feature_with_default_value(self): mock_feature = MockFeature() @@ -167,39 +170,43 @@ def test_enable_feature_with_no_rollback_flag(self): def test_ignore_disable_default_value_when_set_explicitly(self): feature_flag = TEST_FEATURE_FLAG - os.environ[feature_flag] = '0' + config_manager.set_env_var(feature_flag, '0') mock_feature = MockFeature() output = [] result = mock_feature.mock_disabled_default_true(output) self.assertEqual(result, 'mock_disabled_default_true') self.assertListEqual(output, ['mock_disabled_default_true']) + config_manager.del_env_var(feature_flag) def test_disable_feature_with_rollback_flag(self): rollback_flag = TEST_FEATURE_FLAG - os.environ[rollback_flag] = '1' + config_manager.set_env_var(rollback_flag, '1') mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_disabled(output) self.assertIsNone(result) self.assertListEqual(output, []) + config_manager.del_env_var(rollback_flag) def test_enable_feature_with_rollback_flag_is_false(self): rollback_flag = TEST_FEATURE_FLAG - os.environ[rollback_flag] = 'false' + config_manager.set_env_var(rollback_flag, 'false') mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_disabled(output) self.assertEqual(result, 'mock_feature_disabled') self.assertListEqual(output, ['mock_feature_disabled']) + config_manager.del_env_var(rollback_flag) def test_ignore_enable_default_value_when_set_explicitly(self): feature_flag = TEST_FEATURE_FLAG - os.environ[feature_flag] = '0' + config_manager.set_env_var(feature_flag, '0') mock_feature = MockFeature() output = [] result = mock_feature.mock_enabled_default_true(output) self.assertIsNone(result) self.assertListEqual(output, []) + config_manager.del_env_var(feature_flag) def test_fail_to_enable_feature_return_default_value(self): mock_feature = MockFeature() @@ -210,12 +217,13 @@ def test_fail_to_enable_feature_return_default_value(self): def test_disable_feature_with_false_flag_return_default_value(self): feature_flag = TEST_FEATURE_FLAG - os.environ[feature_flag] = 'false' + config_manager.set_env_var(feature_flag, 'false') mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_default(output) self.assertEqual(result, FEATURE_DEFAULT) self.assertListEqual(output, []) + config_manager.del_env_var(feature_flag) def test_exception_message_should_not_be_extended_on_success(self): mock_method = MockMethod() @@ -249,11 +257,12 @@ def test_app_settings_not_set_should_return_none(self): def test_app_settings_should_return_value(self): # Set application setting by os.setenv - os.environ.update({TEST_APP_SETTING_NAME: '42'}) + config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') # Try using utility to acquire application setting app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME) self.assertEqual(app_setting, '42') + config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_not_set_should_return_default_value(self): app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, @@ -262,12 +271,13 @@ def test_app_settings_not_set_should_return_default_value(self): def test_app_settings_should_ignore_default_value(self): # Set application setting by os.setenv - os.environ.update({TEST_APP_SETTING_NAME: '42'}) + config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') # Try using utility to acquire application setting app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, 'default') self.assertEqual(app_setting, '42') + config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_should_not_trigger_validator_when_not_set(self): def raise_excpt(value: str): @@ -285,7 +295,7 @@ def parse_int_no_raise(value: str): return False # Set application setting to an invalid value - os.environ.update({TEST_APP_SETTING_NAME: 'invalid'}) + config_manager.set_env_var(TEST_APP_SETTING_NAME, 'invalid') app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, @@ -295,6 +305,7 @@ def parse_int_no_raise(value: str): # Because 'invalid' is not an interger, falls back to default value self.assertEqual(app_setting, '1') + config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_return_setting_value_when_validation_succeed(self): def parse_int_no_raise(value: str): @@ -305,7 +316,7 @@ def parse_int_no_raise(value: str): return False # Set application setting to an invalid value - os.environ.update({TEST_APP_SETTING_NAME: '42'}) + config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, @@ -315,6 +326,7 @@ def parse_int_no_raise(value: str): # Because 'invalid' is not an interger, falls back to default value self.assertEqual(app_setting, '42') + config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_is_python_version(self): # Should pass at least 1 test @@ -361,19 +373,22 @@ def test_get_sdk_version(self): def test_get_sdk_dummy_version(self): """Test if sdk version can get dummy sdk version """ + config_manager.set_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS, 'false') sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertNotEqual(sdk_version, 'dummy') + config_manager.del_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS) def test_get_sdk_dummy_version_with_flag_enabled(self): """Test if sdk version can get dummy sdk version """ - os.environ[PYTHON_EXTENSIONS_RELOAD_FUNCTIONS] = '1' + config_manager.set_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS, '1') sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertEqual(sdk_version, 'dummy') + config_manager.del_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS) def test_valid_script_file_name(self): file_name = 'test.py' @@ -387,6 +402,6 @@ def test_invalid_script_file_name(self): def _unset_feature_flag(self): try: - os.environ.pop(TEST_FEATURE_FLAG) + config_manager.del_env_var(TEST_FEATURE_FLAG) except KeyError: pass diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index 1f32ca7c4..8ac9287a3 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,6 +7,7 @@ from unittest.mock import patch from azure_functions_worker.utils.dependency import DependencyManager +from azure_functions_worker.utils.config_manager import read_config, set_env_var, del_env_var, is_envvar_true from tests.utils import testutils @@ -37,6 +38,7 @@ def setUp(self): self._patch_sys_path.start() self._patch_importer_cache.start() self._patch_modules.start() + read_config("") def tearDown(self): self._patch_environ.stop() @@ -53,7 +55,7 @@ def test_should_not_have_any_paths_initially(self): self.assertEqual(DependencyManager.worker_deps_path, '') def test_initialize_in_linux_consumption(self): - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') sys.path.extend([ '/tmp/functions\\standby\\wwwroot', '/home/site/wwwroot/.python_packages/lib/site-packages', @@ -73,9 +75,10 @@ def test_initialize_in_linux_consumption(self): DependencyManager.worker_deps_path, '/azure-functions-host/workers/python/3.11/LINUX/X64' ) + del_env_var('AzureWebJobsScriptRoot') def test_initialize_in_linux_dedicated(self): - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') sys.path.extend([ '/home/site/wwwroot', '/home/site/wwwroot/.python_packages/lib/site-packages', @@ -94,9 +97,10 @@ def test_initialize_in_linux_dedicated(self): DependencyManager.worker_deps_path, '/azure-functions-host/workers/python/3.11/LINUX/X64' ) + del_env_var('AzureWebJobsScriptRoot') def test_initialize_in_windows_core_tools(self): - os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') sys.path.extend([ 'C:\\Users\\user\\AppData\\Roaming\\npm\\' 'node_modules\\azure-functions-core-tools\\bin\\' @@ -119,32 +123,36 @@ def test_initialize_in_windows_core_tools(self): 'azure-functions-core-tools\\bin\\workers\\python\\3.11\\WINDOWS' '\\X64' ) + del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_no_script_root(self): result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '') def test_get_cx_deps_path_in_script_root_no_sys_path(self): - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '') + del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_script_root_with_sys_path_linux(self): # Test for Python 3.7+ Azure Environment sys.path.append('/home/site/wwwroot/.python_packages/sites/lib/' 'site-packages/') - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '/home/site/wwwroot/.python_packages/sites/' 'lib/site-packages/') + del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_script_root_with_sys_path_windows(self): # Test for Windows Core Tools Environment sys.path.append('C:\\FunctionApp\\sites\\lib\\site-packages') - os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') result = DependencyManager._get_cx_deps_path() self.assertEqual(result, 'C:\\FunctionApp\\sites\\lib\\site-packages') + del_env_var('AzureWebJobsScriptRoot') def test_get_cx_working_dir_no_script_root(self): result = DependencyManager._get_cx_working_dir() @@ -152,17 +160,19 @@ def test_get_cx_working_dir_no_script_root(self): def test_get_cx_working_dir_with_script_root_linux(self): # Test for Azure Environment - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') result = DependencyManager._get_cx_working_dir() self.assertEqual(result, '/home/site/wwwroot') + del_env_var('AzureWebJobsScriptRoot') def test_get_cx_working_dir_with_script_root_windows(self): # Test for Windows Core Tools Environment - os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' + set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') result = DependencyManager._get_cx_working_dir() self.assertEqual(result, 'C:\\FunctionApp') + del_env_var('AzureWebJobsScriptRoot') - @unittest.skipIf(os.environ.get('VIRTUAL_ENV'), + @unittest.skipIf(is_envvar_true('VIRTUAL_ENV'), 'Test is not capable to run in a virtual environment') def test_get_worker_deps_path_with_no_worker_sys_path(self): result = DependencyManager._get_worker_deps_path() @@ -350,6 +360,9 @@ def test_reload_all_modules_from_customer_deps(self): self._customer_func_path, ]) + del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') + del_env_var('AzureWebJobsScriptRoot') + def test_reload_all_namespaces_from_customer_deps(self): """The test simulates a linux consumption environment where the worker transits from placeholder mode to specialized mode. In a very typical @@ -384,6 +397,9 @@ def test_reload_all_namespaces_from_customer_deps(self): self._customer_func_path, ]) + del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') + del_env_var('AzureWebJobsScriptRoot') + def test_remove_from_sys_path(self): sys.path.append(self._customer_deps_path) DependencyManager._remove_from_sys_path(self._customer_deps_path) @@ -516,7 +532,7 @@ def test_clear_path_importer_cache_and_modules_retain_namespace(self): def test_use_worker_dependencies(self): # Setup app settings - os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -531,9 +547,11 @@ def test_use_worker_dependencies(self): os.path.join(self._worker_deps_path, 'common_module') ) + del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') + def test_use_worker_dependencies_disable(self): # Setup app settings - os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' + set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'false') # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -545,6 +563,8 @@ def test_use_worker_dependencies_disable(self): with self.assertRaises(ImportError): import common_module # NoQA + del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') + def test_use_worker_dependencies_default_python_all_versions(self): # Feature should be disabled for all python versions # Setup paths @@ -559,7 +579,7 @@ def test_use_worker_dependencies_default_python_all_versions(self): def test_prioritize_customer_dependencies(self): # Setup app settings - os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -581,9 +601,11 @@ def test_prioritize_customer_dependencies(self): self._customer_func_path, ]) + del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") + def test_prioritize_customer_dependencies_disable(self): # Setup app settings - os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' + set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'false') # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -595,6 +617,8 @@ def test_prioritize_customer_dependencies_disable(self): with self.assertRaises(ImportError): import common_module # NoQA + del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") + def test_prioritize_customer_dependencies_default_all_versions(self): # Feature should be disabled in Python for all versions # Setup paths @@ -623,6 +647,9 @@ def test_prioritize_customer_dependencies_from_working_directory(self): os.path.join(self._customer_func_path, 'func_specific_module') ) + del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') + del_env_var('AzureWebJobsScriptRoot') + def test_remove_module_cache(self): # First import the common_module and create a sys.modules cache sys.path.append(self._customer_deps_path) @@ -644,8 +671,8 @@ def test_remove_module_cache_with_namespace_remain(self): def _initialize_scenario(self): # Setup app settings - os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' - os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' + set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') + set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path From f653191d10f018701dbec0a1dacf07a3c791c793 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 10:27:59 -0600 Subject: [PATCH 11/33] test_dispatcher starting changes --- tests/unittests/test_dispatcher.py | 21 ++++++++++++++------- tests/unittests/test_utilities.py | 2 -- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 5c8ad015d..d9300a81c 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -13,6 +13,7 @@ from azure_functions_worker.constants import PYTHON_THREADPOOL_THREAD_COUNT, \ PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT, \ PYTHON_THREADPOOL_THREAD_COUNT_MAX_37, PYTHON_THREADPOOL_THREAD_COUNT_MIN +from azure_functions_worker.utils import config_manager SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) @@ -152,13 +153,13 @@ async def test_dispatcher_sync_threadpool_set_worker(self): """Test if the sync threadpool maximum worker can be set """ # Configure thread pool max worker - os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: - f'{self._allowed_max_workers}'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, self._allowed_max_workers) async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_invalid_worker_count(self): """Test when sync threadpool maximum worker is set to an invalid value, @@ -170,7 +171,7 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: # Configure thread pool max worker to an invalid value - os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: 'invalid'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, 'invalid') async with self._ctrl as host: await host.init_worker() @@ -179,6 +180,7 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): self._default_workers) mock_logger.warning.assert_any_call( '%s must be an integer', PYTHON_THREADPOOL_THREAD_COUNT) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_below_min_setting(self): """Test if the sync threadpool will pick up default value when the @@ -187,6 +189,7 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: # Configure thread pool max worker to an invalid value os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '0'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '0') async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) @@ -197,6 +200,7 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): 'Reverting to default value for max_workers', PYTHON_THREADPOOL_THREAD_COUNT, PYTHON_THREADPOOL_THREAD_COUNT_MIN) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """Test if the sync threadpool will pick up default max value when the @@ -204,8 +208,7 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """ with patch('azure_functions_worker.dispatcher.logger'): # Configure thread pool max worker to an invalid value - os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: - f'{self._over_max_workers}'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, self._over_max_workers) async with self._ctrl as host: await host.init_worker('4.15.1') await self._check_if_function_is_ok(host) @@ -213,6 +216,7 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): # Ensure the dispatcher sync threadpool should fallback to max await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_in_placeholder(self): """Test if the sync threadpool will pick up app setting in placeholder @@ -338,7 +342,8 @@ async def test_async_invocation_request_log(self): async def test_sync_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: - os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '5'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '5') + async with self._ctrl as host: await host.init_worker() @@ -360,10 +365,11 @@ async def test_sync_invocation_request_log_threads(self): r'\d{2}:\d{2}:\d{2}.\d{6}), ' 'sync threadpool max workers: 5' ) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: - os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '4'}) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '4') async with self._ctrl as host: await host.init_worker() @@ -384,6 +390,7 @@ async def test_async_invocation_request_log_threads(self): r'(\d{4}-\d{2}-\d{2} ' r'\d{2}:\d{2}:\d{2}.\d{6})' ) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_sync_invocation_request_log_in_placeholder_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 499c1d778..1b77d51d4 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -373,12 +373,10 @@ def test_get_sdk_version(self): def test_get_sdk_dummy_version(self): """Test if sdk version can get dummy sdk version """ - config_manager.set_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS, 'false') sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertNotEqual(sdk_version, 'dummy') - config_manager.del_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS) def test_get_sdk_dummy_version_with_flag_enabled(self): """Test if sdk version can get dummy sdk version From cfe38ca21180448cead425b7c8aa450d0c6ce59c Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 11:59:19 -0600 Subject: [PATCH 12/33] unit tests + lint --- azure_functions_worker/dispatcher.py | 2 +- .../utils/config_manager.py | 1 - azure_functions_worker/utils/dependency.py | 4 +-- tests/unittests/test_dispatcher.py | 28 ++++++++++------ .../test_enable_debug_logging_functions.py | 24 ++++---------- tests/unittests/test_extension.py | 17 ++++++---- tests/unittests/test_file_accessor_factory.py | 12 +++---- tests/unittests/test_loader.py | 4 ++- tests/unittests/test_script_file_name.py | 26 +++++++-------- tests/unittests/test_shared_memory_manager.py | 32 ++++++------------- .../test_third_party_http_functions.py | 11 +++---- tests/unittests/test_utilities_dependency.py | 4 ++- 12 files changed, 77 insertions(+), 88 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index e505955aa..2f679637d 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -294,7 +294,7 @@ async def _handle__worker_init_request(self, request): # Can detech worker packages only when customer's code is present # This only works in dedicated and premium sku. # The consumption sku will switch on environment_reload request. - if not DependencyManager.is_in_linux_consumption(): + if DependencyManager.should_load_cx_dependencies(): DependencyManager.prioritize_customer_dependencies() if DependencyManager.is_in_linux_consumption(): diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index 41cdad9c1..a57dbeb0b 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -100,7 +100,6 @@ def get_app_setting( def set_env_var(setting: str, value: str): global config_data config_data[setting] = value - test = "" def del_env_var(setting: str): diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 000349dc2..625ca3224 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -74,7 +74,7 @@ def initialize(cls): @classmethod def is_in_linux_consumption(cls): - return CONTAINER_NAME in os.environ + return get_app_setting(CONTAINER_NAME) is not None @classmethod def should_load_cx_dependencies(cls): @@ -195,7 +195,7 @@ def reload_customer_libraries(cls, cx_working_dir: str): The path which contains customer's project file (e.g. wwwroot). """ use_new_env = is_envvar_true(PYTHON_ISOLATE_WORKER_DEPENDENCIES) - if use_new_env is None: + if use_new_env is False: use_new = ( PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if is_python_version('3.10') else diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index d9300a81c..b0b199267 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -153,7 +153,8 @@ async def test_dispatcher_sync_threadpool_set_worker(self): """Test if the sync threadpool maximum worker can be set """ # Configure thread pool max worker - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, self._allowed_max_workers) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, + self._allowed_max_workers) async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) @@ -208,7 +209,8 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """ with patch('azure_functions_worker.dispatcher.logger'): # Configure thread pool max worker to an invalid value - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, self._over_max_workers) + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, + self._over_max_workers) async with self._ctrl as host: await host.init_worker('4.15.1') await self._check_if_function_is_ok(host) @@ -344,7 +346,6 @@ async def test_sync_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '5') - async with self._ctrl as host: await host.init_worker() request_id: str = self._ctrl._worker._request_id @@ -684,7 +685,7 @@ async def test_dispatcher_load_azfunc_in_init(self): async def test_dispatcher_load_modules_dedicated_app(self): """Test modules are loaded in dedicated apps """ - os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" + config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") # Dedicated Apps where placeholder mode is not set async with self._ctrl as host: @@ -696,15 +697,16 @@ async def test_dispatcher_load_modules_dedicated_app(self): "working_directory: , Linux Consumption: False," " Placeholder: False", logs ) + config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") async def test_dispatcher_load_modules_con_placeholder_enabled(self): """Test modules are loaded in consumption apps with placeholder mode enabled. """ # Consumption apps with placeholder mode enabled - os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" - os.environ["CONTAINER_NAME"] = "test" - os.environ["WEBSITE_PLACEHOLDER_MODE"] = "1" + config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") + config_manager.set_env_var("CONTAINER_NAME", "test") + config_manager.set_env_var("WEBSITE_PLACEHOLDER_MODE", "1") async with self._ctrl as host: r = await host.init_worker() logs = [log.message for log in r.logs] @@ -712,6 +714,9 @@ async def test_dispatcher_load_modules_con_placeholder_enabled(self): "Applying prioritize_customer_dependencies: " "worker_dependencies_path: , customer_dependencies_path: , " "working_directory: , Linux Consumption: True,", logs) + config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") + config_manager.del_env_var("CONTAINER_NAME") + config_manager.del_env_var("WEBSITE_PLACEHOLDER_MODE") async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): """Test modules are loaded in consumption apps with placeholder mode @@ -719,9 +724,9 @@ async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): """ # Consumption apps with placeholder mode disabled i.e. worker # is specialized - os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" - os.environ["WEBSITE_PLACEHOLDER_MODE"] = "0" - os.environ["CONTAINER_NAME"] = "test" + config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") + config_manager.set_env_var("CONTAINER_NAME", "test") + config_manager.set_env_var("WEBSITE_PLACEHOLDER_MODE", "0") async with self._ctrl as host: r = await host.init_worker() logs = [log.message for log in r.logs] @@ -730,3 +735,6 @@ async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): "worker_dependencies_path: , customer_dependencies_path: , " "working_directory: , Linux Consumption: True," " Placeholder: False", logs) + config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") + config_manager.del_env_var("CONTAINER_NAME") + config_manager.del_env_var("WEBSITE_PLACEHOLDER_MODE") diff --git a/tests/unittests/test_enable_debug_logging_functions.py b/tests/unittests/test_enable_debug_logging_functions.py index 120d54dfe..0a2b86ece 100644 --- a/tests/unittests/test_enable_debug_logging_functions.py +++ b/tests/unittests/test_enable_debug_logging_functions.py @@ -2,10 +2,10 @@ # Licensed under the MIT License. import typing import os -from unittest.mock import patch from tests.utils import testutils from azure_functions_worker.constants import PYTHON_ENABLE_DEBUG_LOGGING +from azure_functions_worker.utils import config_manager from tests.utils.testutils import TESTS_ROOT, remove_path HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO = """\ @@ -27,17 +27,13 @@ class TestDebugLoggingEnabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - os_environ = os.environ.copy() - os_environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' - cls._patch_environ = patch.dict('os.environ', os_environ) - cls._patch_environ.start() + config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '1') super().setUpClass() @classmethod def tearDownClass(cls): - os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) + config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() - cls._patch_environ.stop() @classmethod def get_script_dir(cls): @@ -65,17 +61,13 @@ class TestDebugLoggingDisabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - os_environ = os.environ.copy() - os_environ[PYTHON_ENABLE_DEBUG_LOGGING] = '0' - cls._patch_environ = patch.dict('os.environ', os_environ) - cls._patch_environ.start() + config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '0') super().setUpClass() @classmethod def tearDownClass(cls): - os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) + config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() - cls._patch_environ.stop() @classmethod def get_script_dir(cls): @@ -109,10 +101,7 @@ def setUpClass(cls): with open(host_json, 'w+') as f: f.write(HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO) - os_environ = os.environ.copy() - os_environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' - cls._patch_environ = patch.dict('os.environ', os_environ) - cls._patch_environ.start() + config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '1') super().setUpClass() @classmethod @@ -122,7 +111,6 @@ def tearDownClass(cls): os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() - cls._patch_environ.stop() @classmethod def get_script_dir(cls): diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index c0b390f3f..be3823a8b 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -19,6 +19,7 @@ APP_EXT_POST_INVOCATION, FUNC_EXT_POST_INVOCATION ) from azure_functions_worker.utils.common import get_sdk_from_sys_path +from azure_functions_worker.utils import config_manager class MockContext: @@ -69,10 +70,10 @@ def setUp(self): ) # Set feature flag to on - os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'true' + config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'true') def tearDown(self) -> None: - os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) + config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) self.mock_sys_path.stop() self.mock_sys_module.stop() @@ -136,12 +137,13 @@ def test_function_load_extension_disable_when_feature_flag_is_off( """When turning off the feature flag PYTHON_ENABLE_WORKER_EXTENSIONS, the post_function_load extension should be disabled """ - os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' + config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') self._instance.function_load_extension( func_name=self._mock_func_name, func_directory=self._mock_func_dir ) get_sdk_from_sys_path_mock.assert_not_called() + config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) @patch('azure_functions_worker.extension.ExtensionManager.' '_warn_sdk_not_support_extension') @@ -211,7 +213,7 @@ def test_invocation_extension_extension_disable_when_feature_flag_is_off( """When turning off the feature flag PYTHON_ENABLE_WORKER_EXTENSIONS, the pre_invocation and post_invocation extension should be disabled """ - os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' + config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') self._instance._invocation_extension( ctx=self._mock_context, hook_name=FUNC_EXT_PRE_INVOCATION, @@ -219,6 +221,7 @@ def test_invocation_extension_extension_disable_when_feature_flag_is_off( func_ret=None ) get_sdk_from_sys_path_mock.assert_not_called() + config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) @patch('azure_functions_worker.extension.ExtensionManager.' '_warn_sdk_not_support_extension') @@ -548,7 +551,7 @@ def test_get_sync_invocation_wrapper_disabled_with_flag(self): be executed, but not the extension """ # Turn off feature flag - os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' + config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') # Register a function extension FuncExtClass = self._generate_new_func_extension_class( @@ -571,6 +574,7 @@ def test_get_sync_invocation_wrapper_disabled_with_flag(self): # Ensure the customer's function is executed self.assertEqual(result, 'request_ok') + config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) def test_get_async_invocation_wrapper_no_extension(self): """The async wrapper will wrap an asynchronous function with a @@ -621,7 +625,7 @@ def test_get_invocation_async_disabled_with_flag(self): should not execute the extension. """ # Turn off feature flag - os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' + config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') # Register a function extension FuncExtClass = self._generate_new_func_extension_class( @@ -644,6 +648,7 @@ def test_get_invocation_async_disabled_with_flag(self): # Ensure the customer's function is executed self.assertEqual(result, 'request_ok') + config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) def test_is_pre_invocation_hook(self): for name in (FUNC_EXT_PRE_INVOCATION, APP_EXT_PRE_INVOCATION): diff --git a/tests/unittests/test_file_accessor_factory.py b/tests/unittests/test_file_accessor_factory.py index 3c83f2310..07637297f 100644 --- a/tests/unittests/test_file_accessor_factory.py +++ b/tests/unittests/test_file_accessor_factory.py @@ -4,7 +4,6 @@ import os import sys import unittest -from unittest.mock import patch from azure_functions_worker.bindings.shared_memory_data_transfer \ import FileAccessorFactory @@ -12,6 +11,7 @@ shared_memory_data_transfer.file_accessor_unix import FileAccessorUnix from azure_functions_worker.bindings.\ shared_memory_data_transfer.file_accessor_windows import FileAccessorWindows +from azure_functions_worker.utils import config_manager class TestFileAccessorFactory(unittest.TestCase): @@ -19,13 +19,13 @@ class TestFileAccessorFactory(unittest.TestCase): Tests for FileAccessorFactory. """ def setUp(self): - env = os.environ.copy() - env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" - self.mock_environ = patch.dict('os.environ', env) - self.mock_environ.start() + config_manager.set_env_var( + 'FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED', + ' true') def tearDown(self): - self.mock_environ.stop() + config_manager.del_env_var( + 'FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED') @unittest.skipIf(os.name != 'nt', 'FileAccessorWindows is only valid on Windows') diff --git a/tests/unittests/test_loader.py b/tests/unittests/test_loader.py index 138e78e1b..36454d905 100644 --- a/tests/unittests/test_loader.py +++ b/tests/unittests/test_loader.py @@ -17,6 +17,7 @@ PYTHON_SCRIPT_FILE_NAME_DEFAULT from azure_functions_worker.loader import build_retry_protos from tests.utils import testutils +from azure_functions_worker.utils import config_manager class TestLoader(testutils.WebHostTestCase): @@ -270,7 +271,8 @@ def get_script_dir(cls): 'http_functions_stein' def test_correct_file_name(self): - os.environ.update({PYTHON_SCRIPT_FILE_NAME: self.file_name}) + config_manager.set_env_var(PYTHON_SCRIPT_FILE_NAME, self.file_name) self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), 'function_app.py') + config_manager.del_env_var(PYTHON_SCRIPT_FILE_NAME) diff --git a/tests/unittests/test_script_file_name.py b/tests/unittests/test_script_file_name.py index 43da69f87..3a69aa25d 100644 --- a/tests/unittests/test_script_file_name.py +++ b/tests/unittests/test_script_file_name.py @@ -1,10 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import os from tests.utils import testutils from azure_functions_worker.constants import \ PYTHON_SCRIPT_FILE_NAME, PYTHON_SCRIPT_FILE_NAME_DEFAULT +from azure_functions_worker.utils import config_manager DEFAULT_SCRIPT_FILE_NAME_DIR = testutils.UNIT_TESTS_FOLDER / \ 'file_name_functions' / \ @@ -26,13 +26,13 @@ class TestDefaultScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - os.environ["PYTHON_SCRIPT_FILE_NAME"] = "function_app.py" + config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "function_app.py") super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - os.environ.pop('PYTHON_SCRIPT_FILE_NAME') + config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") super().tearDownClass() @classmethod @@ -43,8 +43,8 @@ def test_default_file_name(self): """ Test the default file name """ - self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), PYTHON_SCRIPT_FILE_NAME_DEFAULT) @@ -55,13 +55,13 @@ class TestNewScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - os.environ["PYTHON_SCRIPT_FILE_NAME"] = "test.py" + config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "test.py") super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - os.environ.pop('PYTHON_SCRIPT_FILE_NAME') + config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") super().tearDownClass() @classmethod @@ -72,8 +72,8 @@ def test_new_file_name(self): """ Test the new file name """ - self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), 'test.py') @@ -84,13 +84,13 @@ class TestInvalidScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - os.environ["PYTHON_SCRIPT_FILE_NAME"] = "main" + config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "main") super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - os.environ.pop('PYTHON_SCRIPT_FILE_NAME') + config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") super().tearDownClass() @classmethod @@ -101,6 +101,6 @@ def test_invalid_file_name(self): """ Test the invalid file name """ - self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), 'main') diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index b5d582945..01ac39ed5 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -2,12 +2,10 @@ # Licensed under the MIT License. import math -import os import json import sys from unittest import skipIf from unittest.mock import patch -from azure_functions_worker.utils.config_manager import is_envvar_true from azure.functions import meta as bind_meta from tests.utils import testutils from azure_functions_worker.bindings.shared_memory_data_transfer \ @@ -16,6 +14,7 @@ import SharedMemoryConstants as consts from azure_functions_worker.constants \ import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED +from azure_functions_worker.utils import config_manager @skipIf(sys.platform == 'darwin', 'MacOS M1 machines do not correctly test the' @@ -26,19 +25,18 @@ class TestSharedMemoryManager(testutils.SharedMemoryTestCase): Tests for SharedMemoryManager. """ def setUp(self): - env = os.environ.copy() - env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" - self.mock_environ = patch.dict('os.environ', env) + config_manager.set_env_var( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, "true") self.mock_sys_module = patch.dict('sys.modules', sys.modules.copy()) self.mock_sys_path = patch('sys.path', sys.path.copy()) - self.mock_environ.start() self.mock_sys_module.start() self.mock_sys_path.start() def tearDown(self): + config_manager.del_env_var( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) self.mock_sys_path.stop() self.mock_sys_module.stop() - self.mock_environ.stop() def test_is_enabled(self): """ @@ -46,17 +44,8 @@ def test_is_enabled(self): enabled. """ - # Make sure shared memory data transfer is enabled - was_shmem_env_true = is_envvar_true( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) - os.environ.update( - {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) manager = SharedMemoryManager() self.assertTrue(manager.is_enabled()) - # Restore the env variable to original value - if not was_shmem_env_true: - os.environ.update( - {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '0'}) def test_is_disabled(self): """ @@ -64,16 +53,13 @@ def test_is_disabled(self): disabled. """ # Make sure shared memory data transfer is disabled - was_shmem_env_true = is_envvar_true( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) - os.environ.update( - {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '0'}) + config_manager.set_env_var( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, '0') manager = SharedMemoryManager() self.assertFalse(manager.is_enabled()) # Restore the env variable to original value - if was_shmem_env_true: - os.environ.update( - {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) + config_manager.set_env_var( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, '1') def test_bytes_input_support(self): """ diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index 634fa2582..573acca95 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -5,7 +5,8 @@ import pathlib import re import typing -from unittest.mock import patch + +from azure_functions_worker.utils import config_manager from tests.utils import testutils from tests.utils.testutils import UNIT_TESTS_ROOT @@ -36,17 +37,15 @@ def setUpClass(cls): host_json = cls.get_script_dir() / 'host.json' with open(host_json, 'w+') as f: f.write(HOST_JSON_TEMPLATE) - os_environ = os.environ.copy() # Turn on feature flag - os_environ['AzureWebJobsFeatureFlags'] = 'EnableWorkerIndexing' - cls._patch_environ = patch.dict('os.environ', os_environ) - cls._patch_environ.start() + config_manager.set_env_var('AzureWebJobsFeatureFlags', + 'EnableWorkerIndexing') super().setUpClass() @classmethod def tearDownClass(cls): + config_manager.del_env_var('AzureWebJobsFeatureFlags') super().tearDownClass() - cls._patch_environ.stop() @classmethod def get_script_dir(cls): diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index 8ac9287a3..ce6d60e59 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,7 +7,9 @@ from unittest.mock import patch from azure_functions_worker.utils.dependency import DependencyManager -from azure_functions_worker.utils.config_manager import read_config, set_env_var, del_env_var, is_envvar_true +from azure_functions_worker.utils.config_manager import (read_config, + set_env_var, del_env_var, + is_envvar_true) from tests.utils import testutils From b77d1c7c7c6926fca0caf833755011fdad5f8a7e Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 13:31:28 -0600 Subject: [PATCH 13/33] filename, db log error --- tests/unittests/test_enable_debug_logging_functions.py | 2 +- tests/unittests/test_loader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_enable_debug_logging_functions.py b/tests/unittests/test_enable_debug_logging_functions.py index 0a2b86ece..4ce82aadc 100644 --- a/tests/unittests/test_enable_debug_logging_functions.py +++ b/tests/unittests/test_enable_debug_logging_functions.py @@ -109,7 +109,7 @@ def tearDownClass(cls): host_json = TESTS_ROOT / cls.get_script_dir() / 'host.json' remove_path(host_json) - os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) + config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() @classmethod diff --git a/tests/unittests/test_loader.py b/tests/unittests/test_loader.py index 36454d905..a9fcb93a7 100644 --- a/tests/unittests/test_loader.py +++ b/tests/unittests/test_loader.py @@ -272,7 +272,7 @@ def get_script_dir(cls): def test_correct_file_name(self): config_manager.set_env_var(PYTHON_SCRIPT_FILE_NAME, self.file_name) - self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), 'function_app.py') config_manager.del_env_var(PYTHON_SCRIPT_FILE_NAME) From 495a2de2b0316f00cba9b0e94b19a3f9c30210fa Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 13:58:59 -0600 Subject: [PATCH 14/33] db logging unit test --- azure_functions_worker/utils/config_manager.py | 5 +++++ .../unittests/test_enable_debug_logging_functions.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index a57dbeb0b..e9f38dadd 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -42,6 +42,11 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: + # special case for PYTHON_ENABLE_DEBUG_LOGGING + # This is read by the host and must be set in os.environ + if key is 'PYTHON_ENABLE_DEBUG_LOGGING': + val = os.getenv(key) + return is_true_like(val) if config_exists() and config_data.get(key) is not None: return is_true_like(config_data.get(key)) return False diff --git a/tests/unittests/test_enable_debug_logging_functions.py b/tests/unittests/test_enable_debug_logging_functions.py index 4ce82aadc..a5e29e524 100644 --- a/tests/unittests/test_enable_debug_logging_functions.py +++ b/tests/unittests/test_enable_debug_logging_functions.py @@ -27,12 +27,12 @@ class TestDebugLoggingEnabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '1') + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' super().setUpClass() @classmethod def tearDownClass(cls): - config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) + os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() @classmethod @@ -61,12 +61,12 @@ class TestDebugLoggingDisabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '0') + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '0' super().setUpClass() @classmethod def tearDownClass(cls): - config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) + os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() @classmethod @@ -101,7 +101,7 @@ def setUpClass(cls): with open(host_json, 'w+') as f: f.write(HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO) - config_manager.set_env_var(PYTHON_ENABLE_DEBUG_LOGGING, '1') + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' super().setUpClass() @classmethod @@ -109,7 +109,7 @@ def tearDownClass(cls): host_json = TESTS_ROOT / cls.get_script_dir() / 'host.json' remove_path(host_json) - config_manager.del_env_var(PYTHON_ENABLE_DEBUG_LOGGING) + os.environ.pop(PYTHON_ENABLE_DEBUG_LOGGING) super().tearDownClass() @classmethod From ca0385512bb01be8e928b5815027e9eb7449ac5d Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 14:43:50 -0600 Subject: [PATCH 15/33] lint --- azure_functions_worker/utils/config_manager.py | 2 +- tests/unittests/test_dispatcher.py | 3 +++ tests/unittests/test_enable_debug_logging_functions.py | 1 - tests/unittests/test_loader.py | 1 - 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index e9f38dadd..b2c60e39f 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -44,7 +44,7 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: # special case for PYTHON_ENABLE_DEBUG_LOGGING # This is read by the host and must be set in os.environ - if key is 'PYTHON_ENABLE_DEBUG_LOGGING': + if key == 'PYTHON_ENABLE_DEBUG_LOGGING': val = os.getenv(key) return is_true_like(val) if config_exists() and config_data.get(key) is not None: diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index b0b199267..6f3fccbbb 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -298,6 +298,8 @@ async def test_dispatcher_sync_threadpool_in_placeholder_below_min(self): async def test_sync_invocation_request_log(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, + PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT) async with self._ctrl as host: await host.init_worker() request_id: str = self._ctrl._worker._request_id @@ -319,6 +321,7 @@ async def test_sync_invocation_request_log(self): 'sync threadpool max workers: ' f'{self._default_workers}' ) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: diff --git a/tests/unittests/test_enable_debug_logging_functions.py b/tests/unittests/test_enable_debug_logging_functions.py index a5e29e524..1f083ea55 100644 --- a/tests/unittests/test_enable_debug_logging_functions.py +++ b/tests/unittests/test_enable_debug_logging_functions.py @@ -5,7 +5,6 @@ from tests.utils import testutils from azure_functions_worker.constants import PYTHON_ENABLE_DEBUG_LOGGING -from azure_functions_worker.utils import config_manager from tests.utils.testutils import TESTS_ROOT, remove_path HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO = """\ diff --git a/tests/unittests/test_loader.py b/tests/unittests/test_loader.py index a9fcb93a7..dbada9332 100644 --- a/tests/unittests/test_loader.py +++ b/tests/unittests/test_loader.py @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import asyncio -import os import pathlib import subprocess import sys From de9d40de117612e5c52166f8b341d95520cd03e2 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 14:55:22 -0600 Subject: [PATCH 16/33] dispatcher test --- tests/unittests/test_dispatcher.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 6f3fccbbb..719c3bba0 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -143,11 +143,14 @@ async def test_dispatcher_sync_threadpool_default_worker(self): """Test if the sync threadpool has maximum worker count set the correct default value """ + config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, + PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT) async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._default_workers) + config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_set_worker(self): """Test if the sync threadpool maximum worker can be set From e975404b2a6fae761cc147c8d383d311461b5d7e Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 6 Mar 2024 15:21:43 -0600 Subject: [PATCH 17/33] weird merge --- azure_functions_worker/dispatcher.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 2f679637d..82fa4b06c 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -291,9 +291,6 @@ async def _handle__worker_init_request(self, request): read_config(os.path.join(worker_init_request.function_app_directory, "az-config.json")) - # Can detech worker packages only when customer's code is present - # This only works in dedicated and premium sku. - # The consumption sku will switch on environment_reload request. if DependencyManager.should_load_cx_dependencies(): DependencyManager.prioritize_customer_dependencies() From 3ff50585bb9376145f1cce10375b309d06b25d2e Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 26 Aug 2024 14:34:47 -0500 Subject: [PATCH 18/33] lint, test refactor, import ref fixes --- .../file_accessor_factory.py | 2 - .../file_accessor_unix.py | 3 - .../shared_memory_manager.py | 5 - azure_functions_worker/http_v2.py | 2 +- .../test_dependency_isolation_functions.py | 3 - tests/endtoend/test_durable_functions.py | 2 - .../endtoend/test_eventhub_batch_functions.py | 2 - tests/endtoend/test_generic_functions.py | 2 +- tests/endtoend/test_servicebus_functions.py | 2 +- tests/endtoend/test_table_functions.py | 3 - tests/endtoend/test_warmup_functions.py | 2 +- tests/unittests/test_dispatcher.py | 272 +++++++++++++++++- .../test_enable_debug_logging_functions.py | 7 +- tests/unittests/test_extension.py | 17 +- tests/unittests/test_file_accessor_factory.py | 12 +- tests/unittests/test_loader.py | 10 +- tests/unittests/test_script_file_name.py | 27 +- tests/unittests/test_shared_memory_manager.py | 32 ++- .../test_third_party_http_functions.py | 10 +- tests/unittests/test_utilities.py | 37 +-- tests/unittests/test_utilities_dependency.py | 61 ++-- tests/utils/testutils.py | 6 - 22 files changed, 361 insertions(+), 158 deletions(-) diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py index 144030808..f6ed531a1 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py @@ -4,8 +4,6 @@ import os import sys -from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED -from ...utils.common import is_envvar_true from .file_accessor import DummyFileAccessor from .file_accessor_unix import FileAccessorUnix from .file_accessor_windows import FileAccessorWindows diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py index 1dc728433..4aebbb219 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py @@ -8,12 +8,9 @@ from azure_functions_worker import constants -from ...logging import logger -from ...utils.common import get_app_setting from .file_accessor import FileAccessor from .shared_memory_constants import SharedMemoryConstants as consts from .shared_memory_exception import SharedMemoryException -from .file_accessor import FileAccessor from ...utils.config_manager import get_app_setting from ...logging import logger diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py index 3edbe887d..27e95952f 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py @@ -4,13 +4,8 @@ import uuid from typing import Dict, Optional -from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED -from ...logging import logger -from ...utils.common import is_envvar_true -from ..datumdef import Datum from .file_accessor_factory import FileAccessorFactory from .shared_memory_constants import SharedMemoryConstants as consts -from .shared_memory_map import SharedMemoryMap from .shared_memory_metadata import SharedMemoryMetadata from .shared_memory_map import SharedMemoryMap from ..datumdef import Datum diff --git a/azure_functions_worker/http_v2.py b/azure_functions_worker/http_v2.py index 2aaf18bda..c4292c21e 100644 --- a/azure_functions_worker/http_v2.py +++ b/azure_functions_worker/http_v2.py @@ -14,7 +14,7 @@ X_MS_INVOCATION_ID, ) from azure_functions_worker.logging import logger -from azure_functions_worker.utils.common import is_envvar_false +from azure_functions_worker.utils.config_manager import is_envvar_false # Http V2 Exceptions diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index e68c162da..f4ce902a4 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -15,9 +15,6 @@ ) from azure_functions_worker.utils.config_manager import is_envvar_true -from tests.utils import testutils -from tests.utils.constants import PYAZURE_INTEGRATION_TEST, \ - CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST REQUEST_TIMEOUT_SEC = 5 diff --git a/tests/endtoend/test_durable_functions.py b/tests/endtoend/test_durable_functions.py index 0a3bc141d..94f58285c 100644 --- a/tests/endtoend/test_durable_functions.py +++ b/tests/endtoend/test_durable_functions.py @@ -10,8 +10,6 @@ from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST from azure_functions_worker.utils.config_manager import is_envvar_true -from tests.utils import testutils -from tests.utils.constants import DEDICATED_DOCKER_TEST, CONSUMPTION_DOCKER_TEST @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/endtoend/test_eventhub_batch_functions.py b/tests/endtoend/test_eventhub_batch_functions.py index 208f54ccc..f520253bb 100644 --- a/tests/endtoend/test_eventhub_batch_functions.py +++ b/tests/endtoend/test_eventhub_batch_functions.py @@ -10,8 +10,6 @@ from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST from azure_functions_worker.utils.config_manager import is_envvar_true -from tests.utils import testutils -from tests.utils.constants import DEDICATED_DOCKER_TEST, CONSUMPTION_DOCKER_TEST @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/endtoend/test_generic_functions.py b/tests/endtoend/test_generic_functions.py index 46f839377..9c4efea34 100644 --- a/tests/endtoend/test_generic_functions.py +++ b/tests/endtoend/test_generic_functions.py @@ -7,7 +7,7 @@ from tests.utils import testutils from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/endtoend/test_servicebus_functions.py b/tests/endtoend/test_servicebus_functions.py index c8b691579..25b765206 100644 --- a/tests/endtoend/test_servicebus_functions.py +++ b/tests/endtoend/test_servicebus_functions.py @@ -7,7 +7,7 @@ from tests.utils import testutils from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/endtoend/test_table_functions.py b/tests/endtoend/test_table_functions.py index 236f5f36d..98d176e7f 100644 --- a/tests/endtoend/test_table_functions.py +++ b/tests/endtoend/test_table_functions.py @@ -9,9 +9,6 @@ from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST from azure_functions_worker.utils.config_manager import is_envvar_true -from tests.utils import testutils -from tests.utils.constants import DEDICATED_DOCKER_TEST, \ - CONSUMPTION_DOCKER_TEST @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/endtoend/test_warmup_functions.py b/tests/endtoend/test_warmup_functions.py index b33eee26f..93bca3ecd 100644 --- a/tests/endtoend/test_warmup_functions.py +++ b/tests/endtoend/test_warmup_functions.py @@ -7,7 +7,7 @@ from tests.utils import testutils from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -from azure_functions_worker.utils.common import is_envvar_true +from azure_functions_worker.utils.config_manager import is_envvar_true @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 2b0b8dd66..a0f5b25e7 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -24,10 +24,6 @@ ) from azure_functions_worker.dispatcher import Dispatcher, ContextEnabledTask from azure_functions_worker.version import VERSION -from tests.utils import testutils -from azure_functions_worker.constants import PYTHON_THREADPOOL_THREAD_COUNT, \ - PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT, \ - PYTHON_THREADPOOL_THREAD_COUNT_MAX_37, PYTHON_THREADPOOL_THREAD_COUNT_MIN from azure_functions_worker.utils import config_manager SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", @@ -783,3 +779,271 @@ async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") config_manager.del_env_var("CONTAINER_NAME") config_manager.del_env_var("WEBSITE_PLACEHOLDER_MODE") + + +class TestDispatcherIndexingInInit(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self.dispatcher = testutils.create_dummy_dispatcher() + sys.path.append(str(FUNCTION_APP_DIRECTORY)) + + def tearDown(self): + self.loop.close() + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'true'}) + def test_worker_init_request_with_indexing_enabled(self): + request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(request)) + + self.assertIsNotNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + del sys.modules['function_app'] + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'false'}) + def test_worker_init_request_with_indexing_disabled(self): + request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(request)) + + self.assertIsNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'true'}) + @patch.object(Dispatcher, 'index_functions') + def test_worker_init_request_with_indexing_exception(self, + mock_index_functions): + mock_index_functions.side_effect = Exception("Mocked Exception") + + request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(request)) + + self.assertIsNone(self.dispatcher._function_metadata_result) + self.assertIsNotNone(self.dispatcher._function_metadata_exception) + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'true'}) + def test_functions_metadata_request_with_init_indexing_enabled(self): + init_request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + metadata_request = protos.StreamingMessage( + functions_metadata_request=protos.FunctionsMetadataRequest( + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + init_response = self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(init_request)) + self.assertEqual(init_response.worker_init_response.result.status, + protos.StatusResult.Success) + + metadata_response = self.loop.run_until_complete( + self.dispatcher._handle__functions_metadata_request( + metadata_request)) + + self.assertEqual( + metadata_response.function_metadata_response.result.status, + protos.StatusResult.Success) + self.assertIsNotNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + del sys.modules['function_app'] + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'false'}) + def test_functions_metadata_request_with_init_indexing_disabled(self): + init_request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + metadata_request = protos.StreamingMessage( + functions_metadata_request=protos.FunctionsMetadataRequest( + function_app_directory=str(str(FUNCTION_APP_DIRECTORY)) + ) + ) + + init_response = self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(init_request)) + self.assertEqual(init_response.worker_init_response.result.status, + protos.StatusResult.Success) + self.assertIsNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + metadata_response = self.loop.run_until_complete( + self.dispatcher._handle__functions_metadata_request( + metadata_request)) + + self.assertEqual( + metadata_response.function_metadata_response.result.status, + protos.StatusResult.Success) + self.assertIsNotNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + del sys.modules['function_app'] + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'true'}) + @patch.object(Dispatcher, 'index_functions') + def test_functions_metadata_request_with_indexing_exception( + self, + mock_index_functions): + mock_index_functions.side_effect = Exception("Mocked Exception") + + request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + metadata_request = protos.StreamingMessage( + functions_metadata_request=protos.FunctionsMetadataRequest( + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(request)) + + self.assertIsNone(self.dispatcher._function_metadata_result) + self.assertIsNotNone(self.dispatcher._function_metadata_exception) + + metadata_response = self.loop.run_until_complete( + self.dispatcher._handle__functions_metadata_request( + metadata_request)) + + self.assertEqual( + metadata_response.function_metadata_response.result.status, + protos.StatusResult.Failure) + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'false'}) + def test_dispatcher_indexing_in_load_request(self): + init_request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(init_request)) + + self.assertIsNone(self.dispatcher._function_metadata_result) + + load_request = protos.StreamingMessage( + function_load_request=protos.FunctionLoadRequest( + function_id="http_trigger", + metadata=protos.RpcFunctionMetadata( + directory=str(FUNCTION_APP_DIRECTORY), + properties={METADATA_PROPERTIES_WORKER_INDEXED: "True"} + ))) + + self.loop.run_until_complete( + self.dispatcher._handle__function_load_request(load_request)) + + self.assertIsNotNone(self.dispatcher._function_metadata_result) + self.assertIsNone(self.dispatcher._function_metadata_exception) + + del sys.modules['function_app'] + + @patch.dict(os.environ, {PYTHON_ENABLE_INIT_INDEXING: 'true'}) + @patch.object(Dispatcher, 'index_functions') + def test_dispatcher_indexing_in_load_request_with_exception( + self, + mock_index_functions): + # This is the case when the second worker has an exception in indexing. + # In this case, we save the error in _function_metadata_exception in + # the init request and throw the error when load request is called. + + mock_index_functions.side_effect = Exception("Mocked Exception") + + init_request = protos.StreamingMessage( + worker_init_request=protos.WorkerInitRequest( + host_version="2.3.4", + function_app_directory=str(FUNCTION_APP_DIRECTORY) + ) + ) + + self.loop.run_until_complete( + self.dispatcher._handle__worker_init_request(init_request)) + + self.assertIsNone(self.dispatcher._function_metadata_result) + + load_request = protos.StreamingMessage( + function_load_request=protos.FunctionLoadRequest( + function_id="http_trigger", + metadata=protos.RpcFunctionMetadata( + directory=str(FUNCTION_APP_DIRECTORY), + properties={METADATA_PROPERTIES_WORKER_INDEXED: "True"} + ))) + + response = self.loop.run_until_complete( + self.dispatcher._handle__function_load_request(load_request)) + + self.assertIsNotNone(self.dispatcher._function_metadata_exception) + self.assertEqual( + response.function_load_response.result.exception.message, + "Exception: Mocked Exception") + + +class TestContextEnabledTask(unittest.TestCase): + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + + def test_init_with_context(self): + # Since ContextEnabledTask accepts the context param, + # no errors will be thrown here + num = contextvars.ContextVar('num') + num.set(5) + ctx = contextvars.copy_context() + exception_raised = False + try: + self.loop.set_task_factory( + lambda loop, coro, context=None: ContextEnabledTask( + coro, loop=loop, context=ctx)) + except TypeError: + exception_raised = True + self.assertFalse(exception_raised) + + async def test_init_without_context(self): + # If the context param is not defined, + # no errors will be thrown for backwards compatibility + exception_raised = False + try: + self.loop.set_task_factory( + lambda loop, coro: ContextEnabledTask( + coro, loop=loop)) + except TypeError: + exception_raised = True + self.assertFalse(exception_raised) diff --git a/tests/unittests/test_enable_debug_logging_functions.py b/tests/unittests/test_enable_debug_logging_functions.py index 61188fef2..112c905a2 100644 --- a/tests/unittests/test_enable_debug_logging_functions.py +++ b/tests/unittests/test_enable_debug_logging_functions.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import os import typing -import os from tests.utils import testutils from tests.utils.testutils import TESTS_ROOT, remove_path @@ -29,7 +28,7 @@ class TestDebugLoggingEnabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = "1" super().setUpClass() @classmethod @@ -63,7 +62,7 @@ class TestDebugLoggingDisabledFunctions(testutils.WebHostTestCase): """ @classmethod def setUpClass(cls): - os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '0' + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = "0" super().setUpClass() @classmethod @@ -103,7 +102,7 @@ def setUpClass(cls): with open(host_json, 'w+') as f: f.write(HOST_JSON_TEMPLATE_WITH_LOGLEVEL_INFO) - os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = '1' + os.environ[PYTHON_ENABLE_DEBUG_LOGGING] = "1" super().setUpClass() @classmethod diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index be62cbb7e..62569fefd 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -24,7 +24,6 @@ ExtensionManager, ) from azure_functions_worker.utils.common import get_sdk_from_sys_path -from azure_functions_worker.utils import config_manager class MockContext: @@ -75,10 +74,10 @@ def setUp(self): ) # Set feature flag to on - config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'true') + os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'true' def tearDown(self) -> None: - config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) + os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) self.mock_sys_path.stop() self.mock_sys_module.stop() @@ -142,13 +141,12 @@ def test_function_load_extension_disable_when_feature_flag_is_off( """When turning off the feature flag PYTHON_ENABLE_WORKER_EXTENSIONS, the post_function_load extension should be disabled """ - config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') + os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' self._instance.function_load_extension( func_name=self._mock_func_name, func_directory=self._mock_func_dir ) get_sdk_from_sys_path_mock.assert_not_called() - config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) @patch('azure_functions_worker.extension.ExtensionManager.' '_warn_sdk_not_support_extension') @@ -218,7 +216,7 @@ def test_invocation_extension_extension_disable_when_feature_flag_is_off( """When turning off the feature flag PYTHON_ENABLE_WORKER_EXTENSIONS, the pre_invocation and post_invocation extension should be disabled """ - config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') + os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' self._instance._invocation_extension( ctx=self._mock_context, hook_name=FUNC_EXT_PRE_INVOCATION, @@ -226,7 +224,6 @@ def test_invocation_extension_extension_disable_when_feature_flag_is_off( func_ret=None ) get_sdk_from_sys_path_mock.assert_not_called() - config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) @patch('azure_functions_worker.extension.ExtensionManager.' '_warn_sdk_not_support_extension') @@ -556,7 +553,7 @@ def test_get_sync_invocation_wrapper_disabled_with_flag(self): be executed, but not the extension """ # Turn off feature flag - config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') + os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' # Register a function extension FuncExtClass = self._generate_new_func_extension_class( @@ -579,7 +576,6 @@ def test_get_sync_invocation_wrapper_disabled_with_flag(self): # Ensure the customer's function is executed self.assertEqual(result, 'request_ok') - config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) def test_get_async_invocation_wrapper_no_extension(self): """The async wrapper will wrap an asynchronous function with a @@ -630,7 +626,7 @@ def test_get_invocation_async_disabled_with_flag(self): should not execute the extension. """ # Turn off feature flag - config_manager.set_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS, 'false') + os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'false' # Register a function extension FuncExtClass = self._generate_new_func_extension_class( @@ -653,7 +649,6 @@ def test_get_invocation_async_disabled_with_flag(self): # Ensure the customer's function is executed self.assertEqual(result, 'request_ok') - config_manager.del_env_var(PYTHON_ENABLE_WORKER_EXTENSIONS) def test_is_pre_invocation_hook(self): for name in (FUNC_EXT_PRE_INVOCATION, APP_EXT_PRE_INVOCATION): diff --git a/tests/unittests/test_file_accessor_factory.py b/tests/unittests/test_file_accessor_factory.py index 5583a8efd..13f8fbada 100644 --- a/tests/unittests/test_file_accessor_factory.py +++ b/tests/unittests/test_file_accessor_factory.py @@ -4,6 +4,7 @@ import os import sys import unittest +from unittest.mock import patch from azure_functions_worker.bindings.shared_memory_data_transfer import ( FileAccessorFactory, @@ -14,7 +15,6 @@ from azure_functions_worker.bindings.shared_memory_data_transfer.file_accessor_windows import ( # NoQA FileAccessorWindows, ) -from azure_functions_worker.utils import config_manager class TestFileAccessorFactory(unittest.TestCase): @@ -22,13 +22,13 @@ class TestFileAccessorFactory(unittest.TestCase): Tests for FileAccessorFactory. """ def setUp(self): - config_manager.set_env_var( - 'FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED', - ' true') + env = os.environ.copy() + env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" + self.mock_environ = patch.dict('os.environ', env) + self.mock_environ.start() def tearDown(self): - config_manager.del_env_var( - 'FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED') + self.mock_environ.stop() @unittest.skipIf(os.name != 'nt', 'FileAccessorWindows is only valid on Windows') diff --git a/tests/unittests/test_loader.py b/tests/unittests/test_loader.py index 61e12a878..a6af8faa5 100644 --- a/tests/unittests/test_loader.py +++ b/tests/unittests/test_loader.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import asyncio +import os import pathlib import subprocess import sys @@ -19,8 +20,6 @@ PYTHON_SCRIPT_FILE_NAME_DEFAULT, ) from azure_functions_worker.loader import build_retry_protos -from tests.utils import testutils -from azure_functions_worker.utils import config_manager class TestLoader(testutils.WebHostTestCase): @@ -276,8 +275,7 @@ def get_script_dir(cls): 'http_functions_stein' def test_correct_file_name(self): - config_manager.set_env_var(PYTHON_SCRIPT_FILE_NAME, self.file_name) - self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), + os.environ.update({PYTHON_SCRIPT_FILE_NAME: self.file_name}) + self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), 'function_app.py') - config_manager.del_env_var(PYTHON_SCRIPT_FILE_NAME) diff --git a/tests/unittests/test_script_file_name.py b/tests/unittests/test_script_file_name.py index eb1fff40b..6d969ebeb 100644 --- a/tests/unittests/test_script_file_name.py +++ b/tests/unittests/test_script_file_name.py @@ -1,13 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import os from tests.utils import testutils from azure_functions_worker.constants import ( PYTHON_SCRIPT_FILE_NAME, PYTHON_SCRIPT_FILE_NAME_DEFAULT, ) -from azure_functions_worker.utils import config_manager DEFAULT_SCRIPT_FILE_NAME_DIR = testutils.UNIT_TESTS_FOLDER / \ 'file_name_functions' / \ @@ -29,13 +28,13 @@ class TestDefaultScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "function_app.py") + os.environ[PYTHON_SCRIPT_FILE_NAME] = "function_app.py" super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") + os.environ.pop(PYTHON_SCRIPT_FILE_NAME) super().tearDownClass() @classmethod @@ -46,8 +45,8 @@ def test_default_file_name(self): """ Test the default file name """ - self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), PYTHON_SCRIPT_FILE_NAME_DEFAULT) @@ -58,13 +57,13 @@ class TestNewScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "test.py") + os.environ[PYTHON_SCRIPT_FILE_NAME] = "test.py" super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") + os.environ.pop(PYTHON_SCRIPT_FILE_NAME) super().tearDownClass() @classmethod @@ -75,8 +74,8 @@ def test_new_file_name(self): """ Test the new file name """ - self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), 'test.py') @@ -87,13 +86,13 @@ class TestInvalidScriptFileName(testutils.WebHostTestCase): @classmethod def setUpClass(cls): - config_manager.set_env_var("PYTHON_SCRIPT_FILE_NAME", "main") + os.environ[PYTHON_SCRIPT_FILE_NAME] = "main" super().setUpClass() @classmethod def tearDownClass(cls): # Remove the PYTHON_SCRIPT_FILE_NAME environment variable - config_manager.del_env_var("PYTHON_SCRIPT_FILE_NAME") + os.environ.pop(PYTHON_SCRIPT_FILE_NAME) super().tearDownClass() @classmethod @@ -104,6 +103,6 @@ def test_invalid_file_name(self): """ Test the invalid file name """ - self.assertIsNotNone(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME)) - self.assertEqual(config_manager.get_app_setting(PYTHON_SCRIPT_FILE_NAME), + self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME)) + self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME), 'main') diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index e431c7d3d..857b89c38 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -3,6 +3,7 @@ import json import math +import os import sys from unittest import skipIf from unittest.mock import patch @@ -19,7 +20,7 @@ from azure_functions_worker.constants import ( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, ) -from azure_functions_worker.utils import config_manager +from azure_functions_worker.utils.config_manager import is_envvar_true @skipIf(sys.platform == 'darwin', 'MacOS M1 machines do not correctly test the' @@ -30,18 +31,19 @@ class TestSharedMemoryManager(testutils.SharedMemoryTestCase): Tests for SharedMemoryManager. """ def setUp(self): - config_manager.set_env_var( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, "true") + env = os.environ.copy() + env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" + self.mock_environ = patch.dict('os.environ', env) self.mock_sys_module = patch.dict('sys.modules', sys.modules.copy()) self.mock_sys_path = patch('sys.path', sys.path.copy()) + self.mock_environ.start() self.mock_sys_module.start() self.mock_sys_path.start() def tearDown(self): - config_manager.del_env_var( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) self.mock_sys_path.stop() self.mock_sys_module.stop() + self.mock_environ.stop() def test_is_enabled(self): """ @@ -49,8 +51,17 @@ def test_is_enabled(self): enabled. """ + # Make sure shared memory data transfer is enabled + was_shmem_env_true = is_envvar_true( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) + os.environ.update( + {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) manager = SharedMemoryManager() self.assertTrue(manager.is_enabled()) + # Restore the env variable to original value + if not was_shmem_env_true: + os.environ.update( + {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '0'}) def test_is_disabled(self): """ @@ -58,13 +69,16 @@ def test_is_disabled(self): disabled. """ # Make sure shared memory data transfer is disabled - config_manager.set_env_var( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, '0') + was_shmem_env_true = is_envvar_true( + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) + os.environ.update( + {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '0'}) manager = SharedMemoryManager() self.assertFalse(manager.is_enabled()) # Restore the env variable to original value - config_manager.set_env_var( - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, '1') + if was_shmem_env_true: + os.environ.update( + {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) def test_bytes_input_support(self): """ diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index 5b578ae58..ad8408655 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -6,7 +6,7 @@ import re import typing -from azure_functions_worker.utils import config_manager +from unittest.mock import patch from tests.utils import testutils from tests.utils.testutils import UNIT_TESTS_ROOT @@ -37,15 +37,17 @@ def setUpClass(cls): host_json = cls.get_script_dir() / 'host.json' with open(host_json, 'w+') as f: f.write(HOST_JSON_TEMPLATE) + os_environ = os.environ.copy() # Turn on feature flag - config_manager.set_env_var('AzureWebJobsFeatureFlags', - 'EnableWorkerIndexing') + os_environ['AzureWebJobsFeatureFlags'] = 'EnableWorkerIndexing' + cls._patch_environ = patch.dict('os.environ', os_environ) + cls._patch_environ.start() super().setUpClass() @classmethod def tearDownClass(cls): - config_manager.del_env_var('AzureWebJobsFeatureFlags') super().tearDownClass() + cls._patch_environ.stop() @classmethod def get_script_dir(cls): diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 327b0fafc..09a6cd088 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -115,7 +115,6 @@ def test_is_false_like_rejected(self): def test_is_envvar_true(self): config_manager.set_env_var(TEST_FEATURE_FLAG, 'true') self.assertTrue(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) - config_manager.del_env_var(TEST_FEATURE_FLAG) def test_is_envvar_not_true_on_unset(self): self._unset_feature_flag() @@ -124,7 +123,6 @@ def test_is_envvar_not_true_on_unset(self): def test_is_envvar_false(self): config_manager.set_env_var(TEST_FEATURE_FLAG, 'false') self.assertTrue(config_manager.is_envvar_false(TEST_FEATURE_FLAG)) - config_manager.del_env_var(TEST_FEATURE_FLAG) def test_is_envvar_not_false_on_unset(self): self._unset_feature_flag() @@ -146,13 +144,12 @@ def test_disable_feature_with_default_value(self): def test_enable_feature_with_feature_flag(self): feature_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(feature_flag, '1') + os.environ[feature_flag] = '1' mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_enabled(output) self.assertEqual(result, 'mock_feature_enabled') self.assertListEqual(output, ['mock_feature_enabled']) - config_manager.del_env_var(feature_flag) def test_enable_feature_with_default_value(self): mock_feature = MockFeature() @@ -170,43 +167,39 @@ def test_enable_feature_with_no_rollback_flag(self): def test_ignore_disable_default_value_when_set_explicitly(self): feature_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(feature_flag, '0') + os.environ[feature_flag] = '0' mock_feature = MockFeature() output = [] result = mock_feature.mock_disabled_default_true(output) self.assertEqual(result, 'mock_disabled_default_true') self.assertListEqual(output, ['mock_disabled_default_true']) - config_manager.del_env_var(feature_flag) def test_disable_feature_with_rollback_flag(self): rollback_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(rollback_flag, '1') + os.environ[rollback_flag] = '1' mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_disabled(output) self.assertIsNone(result) self.assertListEqual(output, []) - config_manager.del_env_var(rollback_flag) def test_enable_feature_with_rollback_flag_is_false(self): rollback_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(rollback_flag, 'false') + os.environ[rollback_flag] = 'false' mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_disabled(output) self.assertEqual(result, 'mock_feature_disabled') self.assertListEqual(output, ['mock_feature_disabled']) - config_manager.del_env_var(rollback_flag) def test_ignore_enable_default_value_when_set_explicitly(self): feature_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(feature_flag, '0') + os.environ[feature_flag] = '0' mock_feature = MockFeature() output = [] result = mock_feature.mock_enabled_default_true(output) self.assertIsNone(result) self.assertListEqual(output, []) - config_manager.del_env_var(feature_flag) def test_fail_to_enable_feature_return_default_value(self): mock_feature = MockFeature() @@ -217,13 +210,12 @@ def test_fail_to_enable_feature_return_default_value(self): def test_disable_feature_with_false_flag_return_default_value(self): feature_flag = TEST_FEATURE_FLAG - config_manager.set_env_var(feature_flag, 'false') + os.environ[feature_flag] = 'false' mock_feature = MockFeature() output = [] result = mock_feature.mock_feature_default(output) self.assertEqual(result, FEATURE_DEFAULT) self.assertListEqual(output, []) - config_manager.del_env_var(feature_flag) def test_exception_message_should_not_be_extended_on_success(self): mock_method = MockMethod() @@ -257,12 +249,11 @@ def test_app_settings_not_set_should_return_none(self): def test_app_settings_should_return_value(self): # Set application setting by os.setenv - config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') + os.environ.update({TEST_APP_SETTING_NAME: '42'}) # Try using utility to acquire application setting app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME) self.assertEqual(app_setting, '42') - config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_not_set_should_return_default_value(self): app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, @@ -271,13 +262,12 @@ def test_app_settings_not_set_should_return_default_value(self): def test_app_settings_should_ignore_default_value(self): # Set application setting by os.setenv - config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') + os.environ.update({TEST_APP_SETTING_NAME: '42'}) # Try using utility to acquire application setting app_setting = config_manager.get_app_setting(TEST_APP_SETTING_NAME, 'default') self.assertEqual(app_setting, '42') - config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_should_not_trigger_validator_when_not_set(self): def raise_excpt(value: str): @@ -295,7 +285,7 @@ def parse_int_no_raise(value: str): return False # Set application setting to an invalid value - config_manager.set_env_var(TEST_APP_SETTING_NAME, 'invalid') + os.environ.update({TEST_APP_SETTING_NAME: 'invalid'}) app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, @@ -305,7 +295,6 @@ def parse_int_no_raise(value: str): # Because 'invalid' is not an interger, falls back to default value self.assertEqual(app_setting, '1') - config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_app_settings_return_setting_value_when_validation_succeed(self): def parse_int_no_raise(value: str): @@ -316,7 +305,7 @@ def parse_int_no_raise(value: str): return False # Set application setting to an invalid value - config_manager.set_env_var(TEST_APP_SETTING_NAME, '42') + os.environ.update({TEST_APP_SETTING_NAME: '42'}) app_setting = config_manager.get_app_setting( TEST_APP_SETTING_NAME, @@ -326,7 +315,6 @@ def parse_int_no_raise(value: str): # Because 'invalid' is not an interger, falls back to default value self.assertEqual(app_setting, '42') - config_manager.del_env_var(TEST_APP_SETTING_NAME) def test_is_python_version(self): # Should pass at least 1 test @@ -383,12 +371,11 @@ def test_get_sdk_dummy_version(self): def test_get_sdk_dummy_version_with_flag_enabled(self): """Test if sdk version can get dummy sdk version """ - config_manager.set_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS, '1') + os.environ[PYTHON_EXTENSIONS_RELOAD_FUNCTIONS] = '1' sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertEqual(sdk_version, 'dummy') - config_manager.del_env_var(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS) def test_valid_script_file_name(self): file_name = 'test.py' @@ -401,6 +388,6 @@ def test_invalid_script_file_name(self): def _unset_feature_flag(self): try: - config_manager.del_env_var(TEST_FEATURE_FLAG) + os.environ.pop(TEST_FEATURE_FLAG) except KeyError: pass diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index 5f0ccde63..fedc0687c 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,13 +7,9 @@ from unittest.mock import patch from azure_functions_worker.utils.dependency import DependencyManager -from azure_functions_worker.utils.config_manager import (read_config, - set_env_var, del_env_var, - is_envvar_true) +from azure_functions_worker.utils.config_manager import read_config from tests.utils import testutils -from azure_functions_worker.utils.dependency import DependencyManager - class TestDependencyManager(unittest.TestCase): @@ -59,7 +55,7 @@ def test_should_not_have_any_paths_initially(self): self.assertEqual(DependencyManager.worker_deps_path, '') def test_initialize_in_linux_consumption(self): - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' sys.path.extend([ '/tmp/functions\\standby\\wwwroot', '/home/site/wwwroot/.python_packages/lib/site-packages', @@ -79,10 +75,9 @@ def test_initialize_in_linux_consumption(self): DependencyManager.worker_deps_path, '/azure-functions-host/workers/python/3.11/LINUX/X64' ) - del_env_var('AzureWebJobsScriptRoot') def test_initialize_in_linux_dedicated(self): - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' sys.path.extend([ '/home/site/wwwroot', '/home/site/wwwroot/.python_packages/lib/site-packages', @@ -101,10 +96,9 @@ def test_initialize_in_linux_dedicated(self): DependencyManager.worker_deps_path, '/azure-functions-host/workers/python/3.11/LINUX/X64' ) - del_env_var('AzureWebJobsScriptRoot') def test_initialize_in_windows_core_tools(self): - set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' sys.path.extend([ 'C:\\Users\\user\\AppData\\Roaming\\npm\\' 'node_modules\\azure-functions-core-tools\\bin\\' @@ -127,36 +121,32 @@ def test_initialize_in_windows_core_tools(self): 'azure-functions-core-tools\\bin\\workers\\python\\3.11\\WINDOWS' '\\X64' ) - del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_no_script_root(self): result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '') def test_get_cx_deps_path_in_script_root_no_sys_path(self): - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '') - del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_script_root_with_sys_path_linux(self): # Test for Python 3.7+ Azure Environment sys.path.append('/home/site/wwwroot/.python_packages/sites/lib/' 'site-packages/') - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' result = DependencyManager._get_cx_deps_path() self.assertEqual(result, '/home/site/wwwroot/.python_packages/sites/' 'lib/site-packages/') - del_env_var('AzureWebJobsScriptRoot') def test_get_cx_deps_path_in_script_root_with_sys_path_windows(self): # Test for Windows Core Tools Environment sys.path.append('C:\\FunctionApp\\sites\\lib\\site-packages') - set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' result = DependencyManager._get_cx_deps_path() self.assertEqual(result, 'C:\\FunctionApp\\sites\\lib\\site-packages') - del_env_var('AzureWebJobsScriptRoot') def test_get_cx_working_dir_no_script_root(self): result = DependencyManager._get_cx_working_dir() @@ -164,19 +154,17 @@ def test_get_cx_working_dir_no_script_root(self): def test_get_cx_working_dir_with_script_root_linux(self): # Test for Azure Environment - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' result = DependencyManager._get_cx_working_dir() self.assertEqual(result, '/home/site/wwwroot') - del_env_var('AzureWebJobsScriptRoot') def test_get_cx_working_dir_with_script_root_windows(self): # Test for Windows Core Tools Environment - set_env_var('AzureWebJobsScriptRoot', 'C:\\FunctionApp') + os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp' result = DependencyManager._get_cx_working_dir() self.assertEqual(result, 'C:\\FunctionApp') - del_env_var('AzureWebJobsScriptRoot') - @unittest.skipIf(is_envvar_true('VIRTUAL_ENV'), + @unittest.skipIf(os.environ.get('VIRTUAL_ENV'), 'Test is not capable to run in a virtual environment') def test_get_worker_deps_path_with_no_worker_sys_path(self): result = DependencyManager._get_worker_deps_path() @@ -364,9 +352,6 @@ def test_reload_all_modules_from_customer_deps(self): self._customer_func_path, ]) - del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') - del_env_var('AzureWebJobsScriptRoot') - def test_reload_all_namespaces_from_customer_deps(self): """The test simulates a linux consumption environment where the worker transits from placeholder mode to specialized mode. In a very typical @@ -401,9 +386,6 @@ def test_reload_all_namespaces_from_customer_deps(self): self._customer_func_path, ]) - del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') - del_env_var('AzureWebJobsScriptRoot') - def test_remove_from_sys_path(self): sys.path.append(self._customer_deps_path) DependencyManager._remove_from_sys_path(self._customer_deps_path) @@ -536,7 +518,7 @@ def test_clear_path_importer_cache_and_modules_retain_namespace(self): def test_use_worker_dependencies(self): # Setup app settings - set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -551,11 +533,9 @@ def test_use_worker_dependencies(self): os.path.join(self._worker_deps_path, 'common_module') ) - del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') - def test_use_worker_dependencies_disable(self): # Setup app settings - set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'false') + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -567,8 +547,6 @@ def test_use_worker_dependencies_disable(self): with self.assertRaises(ImportError): import common_module # NoQA - del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') - def test_use_worker_dependencies_default_python_all_versions(self): # Feature should be disabled for all python versions # Setup paths @@ -583,7 +561,7 @@ def test_use_worker_dependencies_default_python_all_versions(self): def test_prioritize_customer_dependencies(self): # Setup app settings - set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -605,11 +583,9 @@ def test_prioritize_customer_dependencies(self): self._customer_func_path, ]) - del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") - def test_prioritize_customer_dependencies_disable(self): # Setup app settings - set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'false') + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false' # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path @@ -621,8 +597,6 @@ def test_prioritize_customer_dependencies_disable(self): with self.assertRaises(ImportError): import common_module # NoQA - del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") - def test_prioritize_customer_dependencies_default_all_versions(self): # Feature should be disabled in Python for all versions # Setup paths @@ -651,9 +625,6 @@ def test_prioritize_customer_dependencies_from_working_directory(self): os.path.join(self._customer_func_path, 'func_specific_module') ) - del_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES') - del_env_var('AzureWebJobsScriptRoot') - def test_remove_module_cache(self): # First import the common_module and create a sys.modules cache sys.path.append(self._customer_deps_path) @@ -785,8 +756,8 @@ def test_newrelic_protobuf_import_scenario_user_deps(self): def _initialize_scenario(self): # Setup app settings - set_env_var('PYTHON_ISOLATE_WORKER_DEPENDENCIES', 'true') - set_env_var('AzureWebJobsScriptRoot', '/home/site/wwwroot') + os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true' + os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot' # Setup paths DependencyManager.worker_deps_path = self._worker_deps_path diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index 7d2890adc..f464099b5 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -63,12 +63,6 @@ ) from azure_functions_worker.utils.config_manager import (is_envvar_true, get_app_setting) -from tests.utils.constants import PYAZURE_WORKER_DIR, \ - PYAZURE_INTEGRATION_TEST, PROJECT_ROOT, WORKER_CONFIG, \ - CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST, PYAZURE_WEBHOST_DEBUG, \ - ARCHIVE_WEBHOST_LOGS, EXTENSIONS_CSPROJ_TEMPLATE -from tests.utils.testutils_docker import WebHostConsumption, WebHostDedicated, \ - DockerConfigs TESTS_ROOT = PROJECT_ROOT / 'tests' E2E_TESTS_FOLDER = pathlib.Path('endtoend') From b1e7511a5b2b9fcf4f5595e48384b8e317d6d9a9 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 26 Aug 2024 15:11:43 -0500 Subject: [PATCH 19/33] merge fixes --- .../file_accessor_factory.py | 4 +- .../file_accessor_unix.py | 4 +- .../shared_memory_manager.py | 10 ++-- azure_functions_worker/utils/dependency.py | 4 +- tests/unittests/test_dispatcher.py | 48 ++++++------------- 5 files changed, 25 insertions(+), 45 deletions(-) diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py index f6ed531a1..6b22847fd 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py @@ -4,11 +4,11 @@ import os import sys +from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED +from ...utils.config_manager import is_envvar_true from .file_accessor import DummyFileAccessor from .file_accessor_unix import FileAccessorUnix from .file_accessor_windows import FileAccessorWindows -from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED -from ...utils.config_manager import is_envvar_true class FileAccessorFactory: diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py index 4aebbb219..15c4e3dd6 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py @@ -8,11 +8,11 @@ from azure_functions_worker import constants +from ...logging import logger +from ...utils.config_manager import get_app_setting from .file_accessor import FileAccessor from .shared_memory_constants import SharedMemoryConstants as consts from .shared_memory_exception import SharedMemoryException -from ...utils.config_manager import get_app_setting -from ...logging import logger class FileAccessorUnix(FileAccessor): diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py index 27e95952f..869295de9 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py @@ -4,14 +4,14 @@ import uuid from typing import Dict, Optional +from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED +from ...logging import logger +from ...utils.config_manager import is_envvar_true +from ..datumdef import Datum from .file_accessor_factory import FileAccessorFactory from .shared_memory_constants import SharedMemoryConstants as consts -from .shared_memory_metadata import SharedMemoryMetadata from .shared_memory_map import SharedMemoryMap -from ..datumdef import Datum -from ...logging import logger -from ...utils.config_manager import is_envvar_true -from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED +from .shared_memory_metadata import SharedMemoryMetadata class SharedMemoryManager: diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index c34ef3829..e8f58a19c 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -194,8 +194,8 @@ def reload_customer_libraries(cls, cx_working_dir: str): cx_working_dir: str The path which contains customer's project file (e.g. wwwroot). """ - use_new_env = is_envvar_true(PYTHON_ISOLATE_WORKER_DEPENDENCIES) - if use_new_env is False: + use_new_env = get_app_setting(PYTHON_ISOLATE_WORKER_DEPENDENCIES) + if use_new_env is None: use_new = ( PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if is_python_version('3.10') else diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index a0f5b25e7..9b9828e99 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -156,27 +156,23 @@ async def test_dispatcher_sync_threadpool_default_worker(self): """Test if the sync threadpool has maximum worker count set the correct default value """ - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT) async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._default_workers) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_set_worker(self): """Test if the sync threadpool maximum worker can be set """ # Configure thread pool max worker - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, - self._allowed_max_workers) + os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: + f'{self._allowed_max_workers}'}) async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_invalid_worker_count(self): """Test when sync threadpool maximum worker is set to an invalid value, @@ -188,7 +184,7 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: # Configure thread pool max worker to an invalid value - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, 'invalid') + os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: 'invalid'}) async with self._ctrl as host: await host.init_worker() @@ -197,7 +193,6 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): self._default_workers) mock_logger.warning.assert_any_call( '%s must be an integer', PYTHON_THREADPOOL_THREAD_COUNT) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_below_min_setting(self): """Test if the sync threadpool will pick up default value when the @@ -206,7 +201,6 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: # Configure thread pool max worker to an invalid value os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '0'}) - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '0') async with self._ctrl as host: await host.init_worker() await self._check_if_function_is_ok(host) @@ -217,7 +211,6 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): 'Reverting to default value for max_workers', PYTHON_THREADPOOL_THREAD_COUNT, PYTHON_THREADPOOL_THREAD_COUNT_MIN) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """Test if the sync threadpool will pick up default max value when the @@ -225,8 +218,8 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """ with patch('azure_functions_worker.dispatcher.logger'): # Configure thread pool max worker to an invalid value - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, - self._over_max_workers) + os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: + f'{self._over_max_workers}'}) async with self._ctrl as host: await host.init_worker('4.15.1') await self._check_if_function_is_ok(host) @@ -234,7 +227,6 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): # Ensure the dispatcher sync threadpool should fallback to max await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_in_placeholder(self): """Test if the sync threadpool will pick up app setting in placeholder @@ -314,8 +306,6 @@ async def test_dispatcher_sync_threadpool_in_placeholder_below_min(self): async def test_sync_invocation_request_log(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT) async with self._ctrl as host: await host.init_worker() request_id: str = self._ctrl._worker._request_id @@ -337,7 +327,6 @@ async def test_sync_invocation_request_log(self): 'sync threadpool max workers: ' f'{self._default_workers}' ) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -363,7 +352,7 @@ async def test_async_invocation_request_log(self): async def test_sync_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '5') + os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '5'}) async with self._ctrl as host: await host.init_worker() @@ -385,11 +374,10 @@ async def test_sync_invocation_request_log_threads(self): r'\d{2}:\d{2}:\d{2}.\d{6}), ' 'sync threadpool max workers: 5' ) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: - config_manager.set_env_var(PYTHON_THREADPOOL_THREAD_COUNT, '4') + os.environ.update({PYTHON_THREADPOOL_THREAD_COUNT: '4'}) async with self._ctrl as host: await host.init_worker() @@ -410,7 +398,6 @@ async def test_async_invocation_request_log_threads(self): r'(\d{4}-\d{2}-\d{2} ' r'\d{2}:\d{2}:\d{2}.\d{6})' ) - config_manager.del_env_var(PYTHON_THREADPOOL_THREAD_COUNT) async def test_sync_invocation_request_log_in_placeholder_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -726,7 +713,7 @@ async def test_dispatcher_indexing_in_init_request(self): async def test_dispatcher_load_modules_dedicated_app(self): """Test modules are loaded in dedicated apps """ - config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") + os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" # Dedicated Apps where placeholder mode is not set async with self._ctrl as host: @@ -738,16 +725,15 @@ async def test_dispatcher_load_modules_dedicated_app(self): "working_directory: , Linux Consumption: False," " Placeholder: False", logs ) - config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") async def test_dispatcher_load_modules_con_placeholder_enabled(self): """Test modules are loaded in consumption apps with placeholder mode enabled. """ # Consumption apps with placeholder mode enabled - config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") - config_manager.set_env_var("CONTAINER_NAME", "test") - config_manager.set_env_var("WEBSITE_PLACEHOLDER_MODE", "1") + os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" + os.environ["CONTAINER_NAME"] = "test" + os.environ["WEBSITE_PLACEHOLDER_MODE"] = "1" async with self._ctrl as host: r = await host.init_worker() logs = [log.message for log in r.logs] @@ -755,9 +741,6 @@ async def test_dispatcher_load_modules_con_placeholder_enabled(self): "Applying prioritize_customer_dependencies: " "worker_dependencies_path: , customer_dependencies_path: , " "working_directory: , Linux Consumption: True,", logs) - config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") - config_manager.del_env_var("CONTAINER_NAME") - config_manager.del_env_var("WEBSITE_PLACEHOLDER_MODE") async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): """Test modules are loaded in consumption apps with placeholder mode @@ -765,9 +748,9 @@ async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): """ # Consumption apps with placeholder mode disabled i.e. worker # is specialized - config_manager.set_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES", "1") - config_manager.set_env_var("CONTAINER_NAME", "test") - config_manager.set_env_var("WEBSITE_PLACEHOLDER_MODE", "0") + os.environ["PYTHON_ISOLATE_WORKER_DEPENDENCIES"] = "1" + os.environ["WEBSITE_PLACEHOLDER_MODE"] = "0" + os.environ["CONTAINER_NAME"] = "test" async with self._ctrl as host: r = await host.init_worker() logs = [log.message for log in r.logs] @@ -776,9 +759,6 @@ async def test_dispatcher_load_modules_con_app_placeholder_disabled(self): "worker_dependencies_path: , customer_dependencies_path: , " "working_directory: , Linux Consumption: True," " Placeholder: False", logs) - config_manager.del_env_var("PYTHON_ISOLATE_WORKER_DEPENDENCIES") - config_manager.del_env_var("CONTAINER_NAME") - config_manager.del_env_var("WEBSITE_PLACEHOLDER_MODE") class TestDispatcherIndexingInInit(unittest.TestCase): From e85a14c278d49186f1ab4a533251f447a5f09e2f Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 30 Aug 2024 10:23:30 -0500 Subject: [PATCH 20/33] fixed config is not none --- azure_functions_worker/dispatcher.py | 618 ++++++++++-------- .../utils/app_setting_manager.py | 41 +- .../utils/config_manager.py | 24 +- tests/unittests/test_dispatcher.py | 1 - 4 files changed, 386 insertions(+), 298 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 4a52a379a..d0f978625 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -59,8 +59,7 @@ ) from .utils.app_setting_manager import get_python_appsetting_state from .utils.common import validate_script_file_name -from .utils.config_manager import (read_config, is_envvar_true, - get_app_setting) +from .utils.config_manager import read_config, is_envvar_true, get_app_setting from .utils.dependency import DependencyManager from .utils.tracing import marshall_exception_trace from .utils.wrappers import disable_feature_by @@ -78,17 +77,23 @@ class DispatcherMeta(type): def current(mcls): disp = mcls.__current_dispatcher__ if disp is None: - raise RuntimeError('no currently running Dispatcher is found') + raise RuntimeError("no currently running Dispatcher is found") return disp class Dispatcher(metaclass=DispatcherMeta): _GRPC_STOP_RESPONSE = object() - def __init__(self, loop: BaseEventLoop, host: str, port: int, - worker_id: str, request_id: str, - grpc_connect_timeout: float, - grpc_max_msg_len: int = -1) -> None: + def __init__( + self, + loop: BaseEventLoop, + host: str, + port: int, + worker_id: str, + request_id: str, + grpc_connect_timeout: float, + grpc_max_msg_len: int = -1, + ) -> None: self._loop = loop self._host = host self._port = port @@ -113,8 +118,8 @@ def __init__(self, loop: BaseEventLoop, host: str, port: int, # For 3.[6|7|8] The default value is 1. # For 3.9, we don't set this value by default but we honor incoming # the app setting. - self._sync_call_tp: concurrent.futures.Executor = ( - self._create_sync_call_tp(self._get_sync_tp_max_workers()) + self._sync_call_tp: concurrent.futures.Executor = self._create_sync_call_tp( + self._get_sync_tp_max_workers() ) self._grpc_connect_timeout: float = grpc_connect_timeout @@ -123,41 +128,49 @@ def __init__(self, loop: BaseEventLoop, host: str, port: int, self._grpc_resp_queue: queue.Queue = queue.Queue() self._grpc_connected_fut = loop.create_future() self._grpc_thread: threading.Thread = threading.Thread( - name='grpc-thread', target=self.__poll_grpc) + name="grpc-thread", target=self.__poll_grpc + ) @staticmethod def get_worker_metadata(): return protos.WorkerMetadata( runtime_name=PYTHON_LANGUAGE_RUNTIME, - runtime_version=f"{sys.version_info.major}." - f"{sys.version_info.minor}", + runtime_version=f"{sys.version_info.major}." f"{sys.version_info.minor}", worker_version=VERSION, worker_bitness=platform.machine(), - custom_properties={}) + custom_properties={}, + ) def get_sync_tp_workers_set(self): """We don't know the exact value of the threadcount set for the Python - 3.9 scenarios (as we'll start passing only None by default), and we - need to get that information. + 3.9 scenarios (as we'll start passing only None by default), and we + need to get that information. - Ref: concurrent.futures.thread.ThreadPoolExecutor.__init__._max_workers + Ref: concurrent.futures.thread.ThreadPoolExecutor.__init__._max_workers """ return self._sync_call_tp._max_workers @classmethod - async def connect(cls, host: str, port: int, worker_id: str, - request_id: str, connect_timeout: float): + async def connect( + cls, + host: str, + port: int, + worker_id: str, + request_id: str, + connect_timeout: float, + ): loop = asyncio.events.get_event_loop() disp = cls(loop, host, port, worker_id, request_id, connect_timeout) disp._grpc_thread.start() await disp._grpc_connected_fut - logger.info('Successfully opened gRPC channel to %s:%s ', host, port) + logger.info("Successfully opened gRPC channel to %s:%s ", host, port) return disp async def dispatch_forever(self): # sourcery skip: swap-if-expression if DispatcherMeta.__current_dispatcher__ is not None: - raise RuntimeError('there can be only one running dispatcher per ' - 'process') + raise RuntimeError( + "there can be only one running dispatcher per " "process" + ) self._old_task_factory = self._loop.get_task_factory() @@ -170,17 +183,20 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression self._grpc_resp_queue.put_nowait( protos.StreamingMessage( request_id=self.request_id, - start_stream=protos.StartStream( - worker_id=self.worker_id))) + start_stream=protos.StartStream(worker_id=self.worker_id), + ) + ) # In Python 3.11+, constructing a task has an optional context # parameter. Allow for this param to be passed to ContextEnabledTask self._loop.set_task_factory( lambda loop, coro, context=None: ContextEnabledTask( - coro, loop=loop, context=context)) + coro, loop=loop, context=context + ) + ) # Detach console logging before enabling GRPC channel logging - logger.info('Detaching console logging.') + logger.info("Detaching console logging.") disable_console_logging() # Attach gRPC logging to the root logger. Since gRPC channel is @@ -188,24 +204,27 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression logging_handler = AsyncLoggingHandler() root_logger = logging.getLogger() - log_level = logging.INFO if not is_envvar_true( - PYTHON_ENABLE_DEBUG_LOGGING) else logging.DEBUG + log_level = ( + logging.INFO + if not is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING) + else logging.DEBUG + ) root_logger.setLevel(log_level) root_logger.addHandler(logging_handler) - logger.info('Switched to gRPC logging.') + logger.info("Switched to gRPC logging.") logging_handler.flush() try: await forever finally: - logger.warning('Detaching gRPC logging due to exception.') + logger.warning("Detaching gRPC logging due to exception.") logging_handler.flush() root_logger.removeHandler(logging_handler) # Reenable console logging when there's an exception enable_console_logging() - logger.warning('Switched to console logging due to exception.') + logger.warning("Switched to console logging due to exception.") finally: DispatcherMeta.__current_dispatcher__ = None @@ -222,8 +241,7 @@ def stop(self) -> None: self._stop_sync_call_tp() - def on_logging(self, record: logging.LogRecord, - formatted_msg: str) -> None: + def on_logging(self, record: logging.LogRecord, formatted_msg: str) -> None: if record.levelno >= logging.CRITICAL: log_level = protos.RpcLog.Critical elif record.levelno >= logging.ERROR: @@ -235,28 +253,29 @@ def on_logging(self, record: logging.LogRecord, elif record.levelno >= logging.DEBUG: log_level = protos.RpcLog.Debug else: - log_level = getattr(protos.RpcLog, 'None') + log_level = getattr(protos.RpcLog, "None") if is_system_log_category(record.name): - log_category = protos.RpcLog.RpcLogCategory.Value('System') + log_category = protos.RpcLog.RpcLogCategory.Value("System") else: # customers using logging will yield 'root' in record.name - log_category = protos.RpcLog.RpcLogCategory.Value('User') + log_category = protos.RpcLog.RpcLogCategory.Value("User") log = dict( level=log_level, message=formatted_msg, category=record.name, - log_category=log_category + log_category=log_category, ) invocation_id = get_current_invocation_id() if invocation_id is not None: - log['invocation_id'] = invocation_id + log["invocation_id"] = invocation_id self._grpc_resp_queue.put_nowait( protos.StreamingMessage( - request_id=self.request_id, - rpc_log=protos.RpcLog(**log))) + request_id=self.request_id, rpc_log=protos.RpcLog(**log) + ) + ) @property def request_id(self) -> str: @@ -270,35 +289,35 @@ def worker_id(self) -> str: @staticmethod def _serialize_exception(exc: Exception): try: - message = f'{type(exc).__name__}: {exc}' + message = f"{type(exc).__name__}: {exc}" except Exception: - message = ('Unhandled exception in function. ' - 'Could not serialize original exception message.') + message = ( + "Unhandled exception in function. " + "Could not serialize original exception message." + ) try: stack_trace = marshall_exception_trace(exc) except Exception: - stack_trace = '' + stack_trace = "" return protos.RpcException(message=message, stack_trace=stack_trace) async def _dispatch_grpc_request(self, request): - content_type = request.WhichOneof('content') - request_handler = getattr(self, f'_handle__{content_type}', None) + content_type = request.WhichOneof("content") + request_handler = getattr(self, f"_handle__{content_type}", None) if request_handler is None: # Don't crash on unknown messages. Some of them can be ignored; # and if something goes really wrong the host can always just # kill the worker's process. - logger.error('unknown StreamingMessage content type %s', - content_type) + logger.error("unknown StreamingMessage content type %s", content_type) return resp = await request_handler(request) self._grpc_resp_queue.put_nowait(resp) def initialize_azure_monitor(self): - """Initializes OpenTelemetry and Azure monitor distro - """ + """Initializes OpenTelemetry and Azure monitor distro""" self.update_opentelemetry_status() try: from azure.monitor.opentelemetry import configure_azure_monitor @@ -319,21 +338,17 @@ def initialize_azure_monitor(self): ), logger_name=get_app_setting( setting=PYTHON_AZURE_MONITOR_LOGGER_NAME, - default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT + default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT, ), ) self._azure_monitor_available = True logger.info("Successfully configured Azure monitor distro.") except ImportError: - logger.exception( - "Cannot import Azure Monitor distro." - ) + logger.exception("Cannot import Azure Monitor distro.") self._azure_monitor_available = False except Exception: - logger.exception( - "Error initializing Azure monitor distro." - ) + logger.exception("Error initializing Azure monitor distro.") self._azure_monitor_available = False def update_opentelemetry_status(self): @@ -349,25 +364,27 @@ def update_opentelemetry_status(self): self._trace_context_propagator = TraceContextTextMapPropagator() except ImportError: - logger.exception( - "Cannot import OpenTelemetry libraries." - ) + logger.exception("Cannot import OpenTelemetry libraries.") async def _handle__worker_init_request(self, request): - logger.info('Received WorkerInitRequest, ' - 'python version %s, ' - 'worker version %s, ' - 'request ID %s. ' - 'App Settings state: %s. ' - 'To enable debug level logging, please refer to ' - 'https://aka.ms/python-enable-debug-logging', - sys.version, - VERSION, - self.request_id, - get_python_appsetting_state() - ) - worker_init_request = request.worker_init_request + read_config( + os.path.join(worker_init_request.function_app_directory, "az-config.json") + ) + logger.info( + "Received WorkerInitRequest, " + "python version %s, " + "worker version %s, " + "request ID %s. " + "App Settings state: %s. " + "To enable debug level logging, please refer to " + "https://aka.ms/python-enable-debug-logging", + sys.version, + VERSION, + self.request_id, + get_python_appsetting_state(), + ) + host_capabilities = worker_init_request.capabilities if constants.FUNCTION_DATA_CACHE in host_capabilities: val = host_capabilities[constants.FUNCTION_DATA_CACHE] @@ -381,10 +398,10 @@ async def _handle__worker_init_request(self, request): constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE, constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } - read_config(os.path.join(worker_init_request.function_app_directory, - "az-config.json")) - if get_app_setting(setting=PYTHON_ENABLE_OPENTELEMETRY, - default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT): + if get_app_setting( + setting=PYTHON_ENABLE_OPENTELEMETRY, + default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, + ): self.initialize_azure_monitor() if self._azure_monitor_available: @@ -404,11 +421,13 @@ async def _handle__worker_init_request(self, request): try: self.load_function_metadata( worker_init_request.function_app_directory, - caller_info="worker_init_request") + caller_info="worker_init_request", + ) if HttpV2Registry.http_v2_enabled(): - capabilities[constants.HTTP_URI] = \ - initialize_http_server(self._host) + capabilities[constants.HTTP_URI] = initialize_http_server( + self._host + ) except HttpServerInitError: raise @@ -420,8 +439,9 @@ async def _handle__worker_init_request(self, request): worker_init_response=protos.WorkerInitResponse( capabilities=capabilities, worker_metadata=self.get_worker_metadata(), - result=protos.StatusResult( - status=protos.StatusResult.Success))) + result=protos.StatusResult(status=protos.StatusResult.Success), + ), + ) async def _handle__worker_status_request(self, request): # Logging is not necessary in this request since the response is used @@ -429,7 +449,8 @@ async def _handle__worker_status_request(self, request): # Having log here will reduce the responsiveness of the worker. return protos.StreamingMessage( request_id=request.request_id, - worker_status_response=protos.WorkerStatusResponse()) + worker_status_response=protos.WorkerStatusResponse(), + ) def load_function_metadata(self, function_app_directory, caller_info): """ @@ -439,22 +460,27 @@ def load_function_metadata(self, function_app_directory, caller_info): """ script_file_name = get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, - default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}') + default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", + ) logger.debug( - 'Received load metadata request from %s, request ID %s, ' - 'script_file_name: %s', - caller_info, self.request_id, script_file_name) + "Received load metadata request from %s, request ID %s, " + "script_file_name: %s", + caller_info, + self.request_id, + script_file_name, + ) validate_script_file_name(script_file_name) - function_path = os.path.join(function_app_directory, - script_file_name) + function_path = os.path.join(function_app_directory, script_file_name) # For V1, the function path will not exist and # return None. self._function_metadata_result = ( - self.index_functions(function_path, function_app_directory)) \ - if os.path.exists(function_path) else None + (self.index_functions(function_path, function_app_directory)) + if os.path.exists(function_path) + else None + ) async def _handle__functions_metadata_request(self, request): metadata_request = request.functions_metadata_request @@ -462,20 +488,21 @@ async def _handle__functions_metadata_request(self, request): script_file_name = get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, - default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}') - function_path = os.path.join(function_app_directory, - script_file_name) + default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", + ) + function_path = os.path.join(function_app_directory, script_file_name) logger.info( - 'Received WorkerMetadataRequest, request ID %s, ' - 'function_path: %s', - self.request_id, function_path) + "Received WorkerMetadataRequest, request ID %s, " "function_path: %s", + self.request_id, + function_path, + ) if not is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( - function_app_directory, - caller_info="functions_metadata_request") + function_app_directory, caller_info="functions_metadata_request" + ) except Exception as ex: self._function_metadata_exception = ex @@ -486,18 +513,22 @@ async def _handle__functions_metadata_request(self, request): result=protos.StatusResult( status=protos.StatusResult.Failure, exception=self._serialize_exception( - self._function_metadata_exception)))) + self._function_metadata_exception + ), + ) + ), + ) else: metadata_result = self._function_metadata_result return protos.StreamingMessage( request_id=request.request_id, function_metadata_response=protos.FunctionMetadataResponse( - use_default_metadata_indexing=False if metadata_result else - True, + use_default_metadata_indexing=False if metadata_result else True, function_metadata_results=metadata_result, - result=protos.StatusResult( - status=protos.StatusResult.Success))) + result=protos.StatusResult(status=protos.StatusResult.Success), + ), + ) async def _handle__function_load_request(self, request): func_request = request.function_load_request @@ -507,17 +538,21 @@ async def _handle__function_load_request(self, request): function_app_directory = function_metadata.directory logger.info( - 'Received WorkerLoadRequest, request ID %s, function_id: %s,' - 'function_name: %s, function_app_directory : %s', - self.request_id, function_id, function_name, - function_app_directory) + "Received WorkerLoadRequest, request ID %s, function_id: %s," + "function_name: %s, function_app_directory : %s", + self.request_id, + function_id, + function_name, + function_app_directory, + ) programming_model = "V2" try: if not self._functions.get_function(function_id): if function_metadata.properties.get( - METADATA_PROPERTIES_WORKER_INDEXED, False): + METADATA_PROPERTIES_WORKER_INDEXED, False + ): # This is for the second worker and above where the worker # indexing is enabled and load request is called without # calling the metadata request. In this case we index the @@ -525,8 +560,8 @@ async def _handle__function_load_request(self, request): try: self.load_function_metadata( - function_app_directory, - caller_info="functions_load_request") + function_app_directory, caller_info="functions_load_request" + ) except Exception as ex: self._function_metadata_exception = ex @@ -543,36 +578,40 @@ async def _handle__function_load_request(self, request): function_name, function_app_directory, func_request.metadata.script_file, - func_request.metadata.entry_point) + func_request.metadata.entry_point, + ) self._functions.add_function( - function_id, func, func_request.metadata) + function_id, func, func_request.metadata + ) try: ExtensionManager.function_load_extension( - function_name, - func_request.metadata.directory + function_name, func_request.metadata.directory ) except Exception as ex: logging.error("Failed to load extensions: ", ex) raise - logger.info('Successfully processed FunctionLoadRequest, ' - 'request ID: %s, ' - 'function ID: %s,' - 'function Name: %s,' - 'programming model: %s', - self.request_id, - function_id, - function_name, - programming_model) + logger.info( + "Successfully processed FunctionLoadRequest, " + "request ID: %s, " + "function ID: %s," + "function Name: %s," + "programming model: %s", + self.request_id, + function_id, + function_name, + programming_model, + ) return protos.StreamingMessage( request_id=self.request_id, function_load_response=protos.FunctionLoadResponse( function_id=function_id, - result=protos.StatusResult( - status=protos.StatusResult.Success))) + result=protos.StatusResult(status=protos.StatusResult.Success), + ), + ) except Exception as ex: return protos.StreamingMessage( @@ -581,7 +620,10 @@ async def _handle__function_load_request(self, request): function_id=function_id, result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex)))) + exception=self._serialize_exception(ex), + ), + ), + ) async def _handle__invocation_request(self, request): invocation_time = datetime.utcnow() @@ -597,31 +639,30 @@ async def _handle__invocation_request(self, request): current_task.set_azure_invocation_id(invocation_id) try: - fi: functions.FunctionInfo = self._functions.get_function( - function_id) + fi: functions.FunctionInfo = self._functions.get_function(function_id) assert fi is not None function_invocation_logs: List[str] = [ - 'Received FunctionInvocationRequest', - f'request ID: {self.request_id}', - f'function ID: {function_id}', - f'function name: {fi.name}', - f'invocation ID: {invocation_id}', + "Received FunctionInvocationRequest", + f"request ID: {self.request_id}", + f"function ID: {function_id}", + f"function name: {fi.name}", + f"invocation ID: {invocation_id}", f'function type: {"async" if fi.is_async else "sync"}', - f'timestamp (UTC): {invocation_time}' + f"timestamp (UTC): {invocation_time}", ] if not fi.is_async: function_invocation_logs.append( - f'sync threadpool max workers: ' - f'{self.get_sync_tp_workers_set()}' + f"sync threadpool max workers: " f"{self.get_sync_tp_workers_set()}" ) - logger.info(', '.join(function_invocation_logs)) + logger.info(", ".join(function_invocation_logs)) args = {} - http_v2_enabled = self._functions.get_function(function_id) \ - .is_http_func and \ - HttpV2Registry.http_v2_enabled() + http_v2_enabled = ( + self._functions.get_function(function_id).is_http_func + and HttpV2Registry.http_v2_enabled() + ) for pb in invoc_request.input_data: pb_type_info = fi.input_types[pb.name] @@ -636,25 +677,25 @@ async def _handle__invocation_request(self, request): trigger_metadata=trigger_metadata, pytype=pb_type_info.pytype, shmem_mgr=self._shmem_mgr, - function_name=self._functions.get_function( - function_id).name, - is_deferred_binding=pb_type_info.deferred_bindings_enabled) + function_name=self._functions.get_function(function_id).name, + is_deferred_binding=pb_type_info.deferred_bindings_enabled, + ) if http_v2_enabled: http_request = await http_coordinator.get_http_request_async( - invocation_id) + invocation_id + ) await sync_http_request(http_request, invoc_request) - args[fi.trigger_metadata.get('param_name')] = http_request + args[fi.trigger_metadata.get("param_name")] = http_request - fi_context = self._get_context(invoc_request, fi.name, - fi.directory) + fi_context = self._get_context(invoc_request, fi.name, fi.directory) # Use local thread storage to store the invocation ID # for a customer's threads fi_context.thread_local_storage.invocation_id = invocation_id if fi.requires_context: - args['context'] = fi_context + args["context"] = fi_context if fi.output_types: for name in fi.output_types: @@ -664,18 +705,22 @@ async def _handle__invocation_request(self, request): if self._azure_monitor_available: self.configure_opentelemetry(fi_context) - call_result = \ - await self._run_async_func(fi_context, fi.func, args) + call_result = await self._run_async_func(fi_context, fi.func, args) else: call_result = await self._loop.run_in_executor( self._sync_call_tp, self._run_sync_func, - invocation_id, fi_context, fi.func, args) + invocation_id, + fi_context, + fi.func, + args, + ) if call_result is not None and not fi.has_return: raise RuntimeError( - f'function {fi.name!r} without a $return binding' - 'returned a non-None value') + f"function {fi.name!r} without a $return binding" + "returned a non-None value" + ) if http_v2_enabled: http_coordinator.set_http_response(invocation_id, call_result) @@ -691,10 +736,13 @@ async def _handle__invocation_request(self, request): continue param_binding = bindings.to_outgoing_param_binding( - out_type_info.binding_name, val, + out_type_info.binding_name, + val, pytype=out_type_info.pytype, - out_name=out_name, shmem_mgr=self._shmem_mgr, - is_function_data_cache_enabled=cache_enabled) + out_name=out_name, + shmem_mgr=self._shmem_mgr, + is_function_data_cache_enabled=cache_enabled, + ) output_data.append(param_binding) return_value = None @@ -713,9 +761,10 @@ async def _handle__invocation_request(self, request): invocation_response=protos.InvocationResponse( invocation_id=invocation_id, return_value=return_value, - result=protos.StatusResult( - status=protos.StatusResult.Success), - output_data=output_data)) + result=protos.StatusResult(status=protos.StatusResult.Success), + output_data=output_data, + ), + ) except Exception as ex: if http_v2_enabled: @@ -727,7 +776,10 @@ async def _handle__invocation_request(self, request): invocation_id=invocation_id, result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex)))) + exception=self._serialize_exception(ex), + ), + ), + ) async def _handle__function_environment_reload_request(self, request): """Only runs on Linux Consumption placeholder specialization. @@ -735,16 +787,17 @@ async def _handle__function_environment_reload_request(self, request): worker init request will be called directly. """ try: - logger.info('Received FunctionEnvironmentReloadRequest, ' - 'request ID: %s, ' - 'App Settings state: %s. ' - 'To enable debug level logging, please refer to ' - 'https://aka.ms/python-enable-debug-logging', - self.request_id, - get_python_appsetting_state()) - - func_env_reload_request = \ - request.function_environment_reload_request + logger.info( + "Received FunctionEnvironmentReloadRequest, " + "request ID: %s, " + "App Settings state: %s. " + "To enable debug level logging, please refer to " + "https://aka.ms/python-enable-debug-logging", + self.request_id, + get_python_appsetting_state(), + ) + + func_env_reload_request = request.function_environment_reload_request directory = func_env_reload_request.function_app_directory # Append function project root to module finding sys.path @@ -759,14 +812,16 @@ async def _handle__function_environment_reload_request(self, request): env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] - read_config(os.path.join( - func_env_reload_request.function_app_directory, - "az-config.json")) + read_config( + os.path.join( + func_env_reload_request.function_app_directory, "az-config.json" + ) + ) # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() - self._sync_call_tp = ( - self._create_sync_call_tp(self._get_sync_tp_max_workers()) + self._sync_call_tp = self._create_sync_call_tp( + self._get_sync_tp_max_workers() ) if is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING): @@ -782,89 +837,93 @@ async def _handle__function_environment_reload_request(self, request): capabilities = {} if get_app_setting( - setting=PYTHON_ENABLE_OPENTELEMETRY, - default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT): + setting=PYTHON_ENABLE_OPENTELEMETRY, + default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, + ): self.initialize_azure_monitor() if self._azure_monitor_available: - capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = ( - _TRUE) + capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = _TRUE if is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( - directory, - caller_info="environment_reload_request") + directory, caller_info="environment_reload_request" + ) if HttpV2Registry.http_v2_enabled(): - capabilities[constants.HTTP_URI] = \ - initialize_http_server(self._host) + capabilities[constants.HTTP_URI] = initialize_http_server( + self._host + ) except HttpServerInitError: raise except Exception as ex: self._function_metadata_exception = ex # Change function app directory - if getattr(func_env_reload_request, - 'function_app_directory', None): - self._change_cwd( - func_env_reload_request.function_app_directory) + if getattr(func_env_reload_request, "function_app_directory", None): + self._change_cwd(func_env_reload_request.function_app_directory) success_response = protos.FunctionEnvironmentReloadResponse( capabilities=capabilities, worker_metadata=self.get_worker_metadata(), - result=protos.StatusResult( - status=protos.StatusResult.Success)) + result=protos.StatusResult(status=protos.StatusResult.Success), + ) return protos.StreamingMessage( request_id=self.request_id, - function_environment_reload_response=success_response) + function_environment_reload_response=success_response, + ) except Exception as ex: failure_response = protos.FunctionEnvironmentReloadResponse( result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex))) + exception=self._serialize_exception(ex), + ) + ) return protos.StreamingMessage( request_id=self.request_id, - function_environment_reload_response=failure_response) + function_environment_reload_response=failure_response, + ) def index_functions(self, function_path: str, function_dir: str): indexed_functions = loader.index_function_app(function_path) logger.info( - "Indexed function app and found %s functions", - len(indexed_functions) + "Indexed function app and found %s functions", len(indexed_functions) ) if indexed_functions: - fx_metadata_results, fx_bindings_logs = ( - loader.process_indexed_function( - self._functions, - indexed_functions, - function_dir)) + fx_metadata_results, fx_bindings_logs = loader.process_indexed_function( + self._functions, indexed_functions, function_dir + ) indexed_function_logs: List[str] = [] indexed_function_bindings_logs = [] for func in indexed_functions: func_binding_logs = fx_bindings_logs.get(func) for binding in func.get_bindings(): - deferred_binding_info = func_binding_logs.get( - binding.name)\ - if func_binding_logs.get(binding.name) else "" - indexed_function_bindings_logs.append(( - binding.type, binding.name, deferred_binding_info)) - - function_log = "Function Name: {}, Function Binding: {}" \ - .format(func.get_function_name(), - indexed_function_bindings_logs) + deferred_binding_info = ( + func_binding_logs.get(binding.name) + if func_binding_logs.get(binding.name) + else "" + ) + indexed_function_bindings_logs.append( + (binding.type, binding.name, deferred_binding_info) + ) + + function_log = "Function Name: {}, Function Binding: {}".format( + func.get_function_name(), indexed_function_bindings_logs + ) indexed_function_logs.append(function_log) logger.info( - 'Successfully processed FunctionMetadataRequest for ' - 'functions: %s. Deferred bindings enabled: %s.', " ".join( - indexed_function_logs), - self._functions.deferred_bindings_enabled()) + "Successfully processed FunctionMetadataRequest for " + "functions: %s. Deferred bindings enabled: %s.", + " ".join(indexed_function_logs), + self._functions.deferred_bindings_enabled(), + ) return fx_metadata_results @@ -890,59 +949,73 @@ async def _handle__close_shared_memory_resources_request(self, request): for map_name in map_names: try: to_delete_resources = not self._function_data_cache_enabled - success = self._shmem_mgr.free_mem_map(map_name, - to_delete_resources) + success = self._shmem_mgr.free_mem_map( + map_name, to_delete_resources + ) results[map_name] = success except Exception as e: - logger.error('Cannot free memory map %s - %s', map_name, e, - exc_info=True) + logger.error( + "Cannot free memory map %s - %s", map_name, e, exc_info=True + ) finally: response = protos.CloseSharedMemoryResourcesResponse( - close_map_results=results) + close_map_results=results + ) return protos.StreamingMessage( request_id=self.request_id, - close_shared_memory_resources_response=response) + close_shared_memory_resources_response=response, + ) def configure_opentelemetry(self, invocation_context): - carrier = {_TRACEPARENT: invocation_context.trace_context.trace_parent, - _TRACESTATE: invocation_context.trace_context.trace_state} + carrier = { + _TRACEPARENT: invocation_context.trace_context.trace_parent, + _TRACESTATE: invocation_context.trace_context.trace_state, + } ctx = self._trace_context_propagator.extract(carrier) self._context_api.attach(ctx) @staticmethod - def _get_context(invoc_request: protos.InvocationRequest, name: str, - directory: str) -> bindings.Context: - """ For more information refer: + def _get_context( + invoc_request: protos.InvocationRequest, name: str, directory: str + ) -> bindings.Context: + """For more information refer: https://aka.ms/azfunc-invocation-context """ trace_context = bindings.TraceContext( invoc_request.trace_context.trace_parent, invoc_request.trace_context.trace_state, - invoc_request.trace_context.attributes) + invoc_request.trace_context.attributes, + ) retry_context = bindings.RetryContext( invoc_request.retry_context.retry_count, invoc_request.retry_context.max_retry_count, - invoc_request.retry_context.exception) + invoc_request.retry_context.exception, + ) return bindings.Context( - name, directory, invoc_request.invocation_id, - _invocation_id_local, trace_context, retry_context) + name, + directory, + invoc_request.invocation_id, + _invocation_id_local, + trace_context, + retry_context, + ) @disable_feature_by(PYTHON_ROLLBACK_CWD_PATH) def _change_cwd(self, new_cwd: str): if os.path.exists(new_cwd): os.chdir(new_cwd) - logger.info('Changing current working directory to %s', new_cwd) + logger.info("Changing current working directory to %s", new_cwd) else: - logger.warning('Directory %s is not found when reloading', new_cwd) + logger.warning("Directory %s is not found when reloading", new_cwd) def _stop_sync_call_tp(self): """Deallocate the current synchronous thread pool and assign self._sync_call_tp to None. If the thread pool does not exist, this will be a no op. """ - if getattr(self, '_sync_call_tp', None): + if getattr(self, "_sync_call_tp", None): self._sync_call_tp.shutdown() self._sync_call_tp = None @@ -952,45 +1025,48 @@ def tp_max_workers_validator(value: str) -> bool: try: int_value = int(value) except ValueError: - logger.warning('%s must be an integer', - PYTHON_THREADPOOL_THREAD_COUNT) + logger.warning("%s must be an integer", PYTHON_THREADPOOL_THREAD_COUNT) return False if int_value < PYTHON_THREADPOOL_THREAD_COUNT_MIN: logger.warning( - '%s must be set to a value between %s and sys.maxint. ' - 'Reverting to default value for max_workers', + "%s must be set to a value between %s and sys.maxint. " + "Reverting to default value for max_workers", PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_THREADPOOL_THREAD_COUNT_MIN) + PYTHON_THREADPOOL_THREAD_COUNT_MIN, + ) return False return True # Starting Python 3.9, worker won't be putting a limit on the # max_workers count in the created threadpool. - default_value = None if sys.version_info.minor == 9 \ - else f'{PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT}' + default_value = ( + None + if sys.version_info.minor == 9 + else f"{PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT}" + ) - max_workers = get_app_setting(setting=PYTHON_THREADPOOL_THREAD_COUNT, - default_value=default_value, - validator=tp_max_workers_validator) + max_workers = get_app_setting( + setting=PYTHON_THREADPOOL_THREAD_COUNT, + default_value=default_value, + validator=tp_max_workers_validator, + ) if sys.version_info.minor <= 7: - max_workers = min(int(max_workers), - PYTHON_THREADPOOL_THREAD_COUNT_MAX_37) + max_workers = min(int(max_workers), PYTHON_THREADPOOL_THREAD_COUNT_MAX_37) # We can box the app setting as int for earlier python versions. return int(max_workers) if max_workers else None def _create_sync_call_tp( - self, max_worker: Optional[int]) -> concurrent.futures.Executor: + self, max_worker: Optional[int] + ) -> concurrent.futures.Executor: """Create a thread pool executor with max_worker. This is a wrapper over ThreadPoolExecutor constructor. Consider calling this method after _stop_sync_call_tp() to ensure only 1 synchronous thread pool is running. """ - return concurrent.futures.ThreadPoolExecutor( - max_workers=max_worker - ) + return concurrent.futures.ThreadPoolExecutor(max_workers=max_worker) def _run_sync_func(self, invocation_id, context, func, params): # This helper exists because we need to access the current @@ -999,8 +1075,7 @@ def _run_sync_func(self, invocation_id, context, func, params): try: if self._azure_monitor_available: self.configure_opentelemetry(context) - return ExtensionManager.get_sync_invocation_wrapper(context, - func)(params) + return ExtensionManager.get_sync_invocation_wrapper(context, func)(params) finally: context.thread_local_storage.invocation_id = None @@ -1012,24 +1087,20 @@ async def _run_async_func(self, context, func, params): def __poll_grpc(self): options = [] if self._grpc_max_msg_len: - options.append(('grpc.max_receive_message_length', - self._grpc_max_msg_len)) - options.append(('grpc.max_send_message_length', - self._grpc_max_msg_len)) + options.append(("grpc.max_receive_message_length", self._grpc_max_msg_len)) + options.append(("grpc.max_send_message_length", self._grpc_max_msg_len)) - channel = grpc.insecure_channel( - f'{self._host}:{self._port}', options) + channel = grpc.insecure_channel(f"{self._host}:{self._port}", options) try: grpc.channel_ready_future(channel).result( - timeout=self._grpc_connect_timeout) + timeout=self._grpc_connect_timeout + ) except Exception as ex: - self._loop.call_soon_threadsafe( - self._grpc_connected_fut.set_exception, ex) + self._loop.call_soon_threadsafe(self._grpc_connected_fut.set_exception, ex) return else: - self._loop.call_soon_threadsafe( - self._grpc_connected_fut.set_result, True) + self._loop.call_soon_threadsafe(self._grpc_connected_fut.set_result, True) stub = protos.FunctionRpcStub(channel) @@ -1045,14 +1116,17 @@ def gen(resp_queue): try: for req in grpc_req_stream: self._loop.call_soon_threadsafe( - self._loop.create_task, self._dispatch_grpc_request(req)) + self._loop.create_task, self._dispatch_grpc_request(req) + ) except Exception as ex: if ex is grpc_req_stream: # Yes, this is how grpc_req_stream iterator exits. return error_logger.exception( - 'unhandled error in gRPC thread. Exception: {0}'.format( - format_exception(ex))) + "unhandled error in gRPC thread. Exception: {0}".format( + format_exception(ex) + ) + ) raise @@ -1073,12 +1147,15 @@ def emit(self, record: LogRecord) -> None: # Logging such of an issue will cause infinite loop of gRPC logging # To mitigate, we should suppress the 2nd level error logging here # and use print function to report exception instead. - print(f'{CONSOLE_LOG_PREFIX} ERROR: {str(runtime_error)}', - file=sys.stderr, flush=True) + print( + f"{CONSOLE_LOG_PREFIX} ERROR: {str(runtime_error)}", + file=sys.stderr, + flush=True, + ) class ContextEnabledTask(asyncio.Task): - AZURE_INVOCATION_ID = '__azure_function_invocation_id__' + AZURE_INVOCATION_ID = "__azure_function_invocation_id__" def __init__(self, coro, loop, context=None): # The context param is only available for 3.11+. If @@ -1090,8 +1167,7 @@ def __init__(self, coro, loop, context=None): current_task = asyncio.current_task(loop) if current_task is not None: - invocation_id = getattr( - current_task, self.AZURE_INVOCATION_ID, None) + invocation_id = getattr(current_task, self.AZURE_INVOCATION_ID, None) if invocation_id is not None: self.set_azure_invocation_id(invocation_id) @@ -1104,13 +1180,13 @@ def get_current_invocation_id() -> Optional[str]: if loop is not None: current_task = asyncio.current_task(loop) if current_task is not None: - task_invocation_id = getattr(current_task, - ContextEnabledTask.AZURE_INVOCATION_ID, - None) + task_invocation_id = getattr( + current_task, ContextEnabledTask.AZURE_INVOCATION_ID, None + ) if task_invocation_id is not None: return task_invocation_id - return getattr(_invocation_id_local, 'invocation_id', None) + return getattr(_invocation_id_local, "invocation_id", None) _invocation_id_local = threading.local() diff --git a/azure_functions_worker/utils/app_setting_manager.py b/azure_functions_worker/utils/app_setting_manager.py index 3d8ccbb45..b71297820 100644 --- a/azure_functions_worker/utils/app_setting_manager.py +++ b/azure_functions_worker/utils/app_setting_manager.py @@ -1,8 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import os import sys +from .config_manager import get_config from ..constants import ( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, PYTHON_ENABLE_DEBUG_LOGGING, @@ -19,17 +19,18 @@ def get_python_appsetting_state(): - current_vars = os.environ.copy() - python_specific_settings = \ - [PYTHON_ROLLBACK_CWD_PATH, - PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_ISOLATE_WORKER_DEPENDENCIES, - PYTHON_ENABLE_DEBUG_LOGGING, - PYTHON_ENABLE_WORKER_EXTENSIONS, - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, - PYTHON_SCRIPT_FILE_NAME, - PYTHON_ENABLE_INIT_INDEXING, - PYTHON_ENABLE_OPENTELEMETRY] + current_vars = get_config() + python_specific_settings = [ + PYTHON_ROLLBACK_CWD_PATH, + PYTHON_THREADPOOL_THREAD_COUNT, + PYTHON_ISOLATE_WORKER_DEPENDENCIES, + PYTHON_ENABLE_DEBUG_LOGGING, + PYTHON_ENABLE_WORKER_EXTENSIONS, + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, + PYTHON_SCRIPT_FILE_NAME, + PYTHON_ENABLE_INIT_INDEXING, + PYTHON_ENABLE_OPENTELEMETRY, + ] app_setting_states = "".join( f"{app_setting}: {current_vars[app_setting]} | " @@ -38,14 +39,16 @@ def get_python_appsetting_state(): ) # Special case for extensions - if 'PYTHON_ENABLE_WORKER_EXTENSIONS' not in current_vars: + if "PYTHON_ENABLE_WORKER_EXTENSIONS" not in current_vars: if sys.version_info.minor == 9: - app_setting_states += \ - (f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " - f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39)}") + app_setting_states += ( + f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " + f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39)}" + ) else: - app_setting_states += \ - (f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " - f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT)}") + app_setting_states += ( + f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " + f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT)}" + ) return app_setting_states diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index b2c60e39f..e0dea4653 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -2,15 +2,18 @@ # Licensed under the MIT License. import os import json -from typing import Optional, Callable +from typing import Optional, Callable, Dict -config_data = {} +# Initialize to None +config_data: Optional[Dict[str, str]] = None def read_config(function_path: str): + global config_data + if config_data is None: + config_data = {} try: with open(function_path, "r") as stream: - global config_data # loads the entire json file full_config_data = json.load(stream) # gets the python section of the json file @@ -24,27 +27,34 @@ def read_config(function_path: str): def config_exists() -> bool: + global config_data + if config_data is None: + read_config("") return config_data is not {} +def get_config() -> dict: + return config_data + + def is_true_like(setting: str) -> bool: if setting is None: return False - return setting.lower().strip() in {'1', 'true', 't', 'yes', 'y'} + return setting.lower().strip() in {"1", "true", "t", "yes", "y"} def is_false_like(setting: str) -> bool: if setting is None: return False - return setting.lower().strip() in {'0', 'false', 'f', 'no', 'n'} + return setting.lower().strip() in {"0", "false", "f", "no", "n"} def is_envvar_true(key: str) -> bool: # special case for PYTHON_ENABLE_DEBUG_LOGGING # This is read by the host and must be set in os.environ - if key == 'PYTHON_ENABLE_DEBUG_LOGGING': + if key == "PYTHON_ENABLE_DEBUG_LOGGING": val = os.getenv(key) return is_true_like(val) if config_exists() and config_data.get(key) is not None: @@ -61,7 +71,7 @@ def is_envvar_false(key: str) -> bool: def get_app_setting( setting: str, default_value: Optional[str] = None, - validator: Optional[Callable[[str], bool]] = None + validator: Optional[Callable[[str], bool]] = None, ) -> Optional[str]: """Returns the application setting from environment variable. diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 9b9828e99..32eca34bf 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -24,7 +24,6 @@ ) from azure_functions_worker.dispatcher import Dispatcher, ContextEnabledTask from azure_functions_worker.version import VERSION -from azure_functions_worker.utils import config_manager SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) From 9250771fa599f1bfd0092aa06a7729b2c3f3aec5 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 30 Aug 2024 15:15:35 -0500 Subject: [PATCH 21/33] local logging (revert) --- azure_functions_worker/dispatcher.py | 6 ++++++ azure_functions_worker/logging.py | 2 ++ azure_functions_worker/utils/config_manager.py | 4 ++++ tests/unittests/test_dispatcher.py | 7 +++++++ tests/utils/testutils.py | 6 +++--- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index d0f978625..1855d364c 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -368,6 +368,11 @@ def update_opentelemetry_status(self): async def _handle__worker_init_request(self, request): worker_init_request = request.worker_init_request + # Apply PYTHON_THREADPOOL_THREAD_COUNT + self._stop_sync_call_tp() + self._sync_call_tp = self._create_sync_call_tp( + self._get_sync_tp_max_workers() + ) read_config( os.path.join(worker_init_request.function_app_directory, "az-config.json") ) @@ -1051,6 +1056,7 @@ def tp_max_workers_validator(value: str) -> bool: default_value=default_value, validator=tp_max_workers_validator, ) + logger.info(f"max_workers: {max_workers}") if sys.version_info.minor <= 7: max_workers = min(int(max_workers), PYTHON_THREADPOOL_THREAD_COUNT_MAX_37) diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py index adb5ff294..8ecb2880d 100644 --- a/azure_functions_worker/logging.py +++ b/azure_functions_worker/logging.py @@ -13,6 +13,7 @@ SDK_LOG_PREFIX = "azure.functions" SYSTEM_ERROR_LOG_PREFIX = "azure_functions_worker_errors" +local_handler = logging.FileHandler("C:\\Users\\victoriahall\\Documents\\repos\\azure-functions-python-worker\\mylog.txt") logger: logging.Logger = logging.getLogger(SYSTEM_LOG_PREFIX) error_logger: logging.Logger = ( @@ -54,6 +55,7 @@ def setup(log_level, log_destination): error_handler = logging.StreamHandler(sys.stderr) error_handler.setFormatter(formatter) error_handler.setLevel(getattr(logging, log_level)) + logger.addHandler(local_handler) handler = logging.StreamHandler(sys.stdout) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index e0dea4653..c4aa71a2d 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -4,6 +4,8 @@ import json from typing import Optional, Callable, Dict +from ..logging import logger + # Initialize to None config_data: Optional[Dict[str, str]] = None @@ -73,6 +75,7 @@ def get_app_setting( default_value: Optional[str] = None, validator: Optional[Callable[[str], bool]] = None, ) -> Optional[str]: + logger.error("Setting: %s, Config data at setting %s", setting, config_data.get(setting)) """Returns the application setting from environment variable. Parameters @@ -109,6 +112,7 @@ def get_app_setting( # Setting is not configured or validator is false # Return default value + logger.info("Returning default value: %s", default_value) return default_value diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 32eca34bf..30cdca28d 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -172,6 +172,7 @@ async def test_dispatcher_sync_threadpool_set_worker(self): await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_invalid_worker_count(self): """Test when sync threadpool maximum worker is set to an invalid value, @@ -192,6 +193,7 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): self._default_workers) mock_logger.warning.assert_any_call( '%s must be an integer', PYTHON_THREADPOOL_THREAD_COUNT) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_below_min_setting(self): """Test if the sync threadpool will pick up default value when the @@ -210,6 +212,7 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): 'Reverting to default value for max_workers', PYTHON_THREADPOOL_THREAD_COUNT, PYTHON_THREADPOOL_THREAD_COUNT_MIN) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """Test if the sync threadpool will pick up default max value when the @@ -226,6 +229,7 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): # Ensure the dispatcher sync threadpool should fallback to max await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_in_placeholder(self): """Test if the sync threadpool will pick up app setting in placeholder @@ -373,6 +377,7 @@ async def test_sync_invocation_request_log_threads(self): r'\d{2}:\d{2}:\d{2}.\d{6}), ' 'sync threadpool max workers: 5' ) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -397,6 +402,7 @@ async def test_async_invocation_request_log_threads(self): r'(\d{4}-\d{2}-\d{2} ' r'\d{2}:\d{2}:\d{2}.\d{6})' ) + os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_sync_invocation_request_log_in_placeholder_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -724,6 +730,7 @@ async def test_dispatcher_load_modules_dedicated_app(self): "working_directory: , Linux Consumption: False," " Placeholder: False", logs ) + os.environ.pop("PYTHON_ISOLATE_WORKER_DEPENDENCIES") async def test_dispatcher_load_modules_con_placeholder_enabled(self): """Test modules are loaded in consumption apps with placeholder mode diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index f464099b5..576e1f640 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -157,7 +157,7 @@ def wrapper(self, *args, __meth__=test_case, __check_log__=check_log_case, **kwargs): if (__check_log__ is not None and callable(__check_log__) - and not is_envvar_true(PYAZURE_WEBHOST_DEBUG)): + and not True): # Check logging output for unit test scenarios result = self._run_test(__meth__, *args, **kwargs) @@ -231,7 +231,7 @@ def setUpClass(cls): docker_tests_enabled, sku = cls.docker_tests_enabled() - cls.host_stdout = None if is_envvar_true(PYAZURE_WEBHOST_DEBUG) \ + cls.host_stdout = None if True \ else tempfile.NamedTemporaryFile('w+t') try: @@ -967,7 +967,7 @@ def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None): def start_webhost(*, script_dir=None, stdout=None): script_root = TESTS_ROOT / script_dir if script_dir else FUNCS_PATH if stdout is None: - if is_envvar_true(PYAZURE_WEBHOST_DEBUG): + if True: stdout = sys.stdout else: stdout = subprocess.DEVNULL From 2f207715e7abf0df134e0e950f3864c26ee57763 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 3 Sep 2024 15:28:10 -0500 Subject: [PATCH 22/33] unit test fixes --- .../utils/config_manager.py | 32 ++++++++++++------- tests/unittests/test_dispatcher.py | 2 ++ tests/unittests/test_shared_memory_manager.py | 2 ++ tests/unittests/test_utilities.py | 5 +-- tests/unittests/test_utilities_dependency.py | 4 +-- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index e0dea4653..ddc62ad3f 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -30,7 +30,7 @@ def config_exists() -> bool: global config_data if config_data is None: read_config("") - return config_data is not {} + return config_data is not None def get_config() -> dict: @@ -52,20 +52,22 @@ def is_false_like(setting: str) -> bool: def is_envvar_true(key: str) -> bool: + key_upper = key.upper() # special case for PYTHON_ENABLE_DEBUG_LOGGING # This is read by the host and must be set in os.environ - if key == "PYTHON_ENABLE_DEBUG_LOGGING": - val = os.getenv(key) + if key_upper == "PYTHON_ENABLE_DEBUG_LOGGING": + val = os.getenv(key_upper) return is_true_like(val) - if config_exists() and config_data.get(key) is not None: - return is_true_like(config_data.get(key)) - return False + if config_exists() and not config_data.get(key_upper): + return False + return is_true_like(config_data.get(key_upper)) def is_envvar_false(key: str) -> bool: - if config_exists() and config_data.get(key) is not None: - return is_false_like(config_data.get(key)) - return False + key_upper = key.upper() + if config_exists() and not config_data.get(key_upper): + return False + return is_false_like(config_data.get(key_upper)) def get_app_setting( @@ -93,9 +95,10 @@ def get_app_setting( Optional[str] A string value that is set in the application setting """ - if config_exists() and config_data.get(setting) is not None: + setting_upper = setting.upper() + if config_exists() and config_data.get(setting_upper) is not None: # Setting exists, check with validator - app_setting_value = config_data.get(setting) + app_setting_value = config_data.get(setting_upper) # If there's no validator, return the app setting value directly if validator is None: @@ -120,3 +123,10 @@ def set_env_var(setting: str, value: str): def del_env_var(setting: str): global config_data config_data.pop(setting, None) + + +def clear_config(): + global config_data + if config_data is not None: + config_data.clear() + config_data = None diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 30cdca28d..f16114ec6 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -23,6 +23,7 @@ PYTHON_THREADPOOL_THREAD_COUNT_MIN, ) from azure_functions_worker.dispatcher import Dispatcher, ContextEnabledTask +from azure_functions_worker.utils import config_manager from azure_functions_worker.version import VERSION SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", @@ -62,6 +63,7 @@ def tearDown(self): os.environ.clear() os.environ.update(self._pre_env) self.mock_version_info.stop() + config_manager.clear_config() async def test_dispatcher_initialize_worker(self): """Test if the dispatcher can be initialized worker successfully diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index 857b89c38..f87bb220f 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -9,6 +9,7 @@ from unittest.mock import patch from azure.functions import meta as bind_meta +from azure_functions_worker.utils import config_manager from tests.utils import testutils from azure_functions_worker.bindings.shared_memory_data_transfer import ( @@ -31,6 +32,7 @@ class TestSharedMemoryManager(testutils.SharedMemoryTestCase): Tests for SharedMemoryManager. """ def setUp(self): + config_manager.clear_config() env = os.environ.copy() env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" self.mock_environ = patch.dict('os.environ', env) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 09a6cd088..34dc9f996 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -87,6 +87,7 @@ def tearDown(self): self.mock_sys_path.stop() self.mock_sys_module.stop() self.mock_environ.stop() + config_manager.clear_config() def test_is_true_like_accepted(self): self.assertTrue(config_manager.is_true_like('1')) @@ -113,7 +114,7 @@ def test_is_false_like_rejected(self): self.assertFalse(config_manager.is_false_like('secret')) def test_is_envvar_true(self): - config_manager.set_env_var(TEST_FEATURE_FLAG, 'true') + os.environ[TEST_FEATURE_FLAG] = 'true' self.assertTrue(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_is_envvar_not_true_on_unset(self): @@ -121,7 +122,7 @@ def test_is_envvar_not_true_on_unset(self): self.assertFalse(config_manager.is_envvar_true(TEST_FEATURE_FLAG)) def test_is_envvar_false(self): - config_manager.set_env_var(TEST_FEATURE_FLAG, 'false') + os.environ[TEST_FEATURE_FLAG] = 'false' self.assertTrue(config_manager.is_envvar_false(TEST_FEATURE_FLAG)) def test_is_envvar_not_false_on_unset(self): diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index fedc0687c..a88203ecf 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,7 +7,7 @@ from unittest.mock import patch from azure_functions_worker.utils.dependency import DependencyManager -from azure_functions_worker.utils.config_manager import read_config +from azure_functions_worker.utils.config_manager import clear_config from tests.utils import testutils @@ -38,7 +38,7 @@ def setUp(self): self._patch_sys_path.start() self._patch_importer_cache.start() self._patch_modules.start() - read_config("") + clear_config() def tearDown(self): self._patch_environ.stop() From 210f5c43a79aac9c9bb5911fcda4bea5bffb0d23 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 4 Sep 2024 11:56:46 -0500 Subject: [PATCH 23/33] refactor all keys to be upper case --- azure_functions_worker/dispatcher.py | 4 +++- azure_functions_worker/utils/config_manager.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 1855d364c..aeb2ff5b9 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -59,7 +59,8 @@ ) from .utils.app_setting_manager import get_python_appsetting_state from .utils.common import validate_script_file_name -from .utils.config_manager import read_config, is_envvar_true, get_app_setting +from .utils.config_manager import (clear_config, read_config, + is_envvar_true, get_app_setting) from .utils.dependency import DependencyManager from .utils.tracing import marshall_exception_trace from .utils.wrappers import disable_feature_by @@ -814,6 +815,7 @@ async def _handle__function_environment_reload_request(self, request): # Reload environment variables os.environ.clear() + clear_config() env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index ddc62ad3f..11b78962f 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -20,10 +20,12 @@ def read_config(function_path: str): config_data = full_config_data.get("PYTHON") except FileNotFoundError: pass - env_copy = os.environ - # updates the config dictionary with the environment variables + + # updates the config dictionary with the environment variables # this prioritizes set env variables over the config file - config_data.update(env_copy) + env_copy = os.environ + for k, v in env_copy.items(): + config_data.update({k.upper(): v}) def config_exists() -> bool: From 033c480cf1503a35f301c94fbdc867448daa47b3 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 5 Sep 2024 11:05:44 -0500 Subject: [PATCH 24/33] unit test fixes --- tests/unittests/test_dispatcher.py | 2 ++ tests/unittests/test_extension.py | 2 ++ tests/unittests/test_opentelemetry.py | 2 ++ tests/unittests/test_shared_memory_manager.py | 3 ++- 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index f16114ec6..d40e1fd95 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -602,6 +602,7 @@ class TestDispatcherStein(testutils.AsyncTestCase): def setUp(self): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_STEIN_FUNCTIONS_DIR) + config_manager.clear_config() async def test_dispatcher_functions_metadata_request(self): """Test if the functions metadata response will be sent correctly @@ -668,6 +669,7 @@ def setUp(self): 'azure_functions_worker.dispatcher.sys.version_info', SysVersionInfo(3, 9, 0, 'final', 0)) self.mock_version_info.start() + config_manager.clear_config() def tearDown(self): os.environ.clear() diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 62569fefd..b10ee184f 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -24,6 +24,7 @@ ExtensionManager, ) from azure_functions_worker.utils.common import get_sdk_from_sys_path +from azure_functions_worker.utils.config_manager import clear_config class MockContext: @@ -75,6 +76,7 @@ def setUp(self): # Set feature flag to on os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'true' + clear_config() def tearDown(self) -> None: os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) diff --git a/tests/unittests/test_opentelemetry.py b/tests/unittests/test_opentelemetry.py index b26334bdf..58121fe68 100644 --- a/tests/unittests/test_opentelemetry.py +++ b/tests/unittests/test_opentelemetry.py @@ -7,6 +7,7 @@ from tests.utils import testutils from azure_functions_worker import protos +from azure_functions_worker.utils import config_manager class TestOpenTelemetry(unittest.TestCase): @@ -15,6 +16,7 @@ def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.dispatcher = testutils.create_dummy_dispatcher() + config_manager.clear_config() def tearDown(self): self.loop.close() diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index f87bb220f..80d3e683d 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -32,7 +32,6 @@ class TestSharedMemoryManager(testutils.SharedMemoryTestCase): Tests for SharedMemoryManager. """ def setUp(self): - config_manager.clear_config() env = os.environ.copy() env['FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED'] = "true" self.mock_environ = patch.dict('os.environ', env) @@ -41,6 +40,7 @@ def setUp(self): self.mock_environ.start() self.mock_sys_module.start() self.mock_sys_path.start() + config_manager.clear_config() def tearDown(self): self.mock_sys_path.stop() @@ -73,6 +73,7 @@ def test_is_disabled(self): # Make sure shared memory data transfer is disabled was_shmem_env_true = is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) + config_manager.clear_config() os.environ.update( {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '0'}) manager = SharedMemoryManager() From 22264c3c82c69c43670937ac33e05eee32607a07 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 5 Sep 2024 14:12:56 -0500 Subject: [PATCH 25/33] fix unit tests --- tests/unittests/test_app_setting_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unittests/test_app_setting_manager.py b/tests/unittests/test_app_setting_manager.py index d203704f9..e1646f9b8 100644 --- a/tests/unittests/test_app_setting_manager.py +++ b/tests/unittests/test_app_setting_manager.py @@ -12,6 +12,7 @@ PYTHON_THREADPOOL_THREAD_COUNT, ) from azure_functions_worker.utils.app_setting_manager import get_python_appsetting_state +from azure_functions_worker.utils.config_manager import clear_config SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) @@ -71,6 +72,7 @@ def setUpClass(cls): cls._patch_environ = patch.dict('os.environ', os_environ) cls._patch_environ.start() super().setUpClass() + clear_config() @classmethod def tearDownClass(cls): From 83d83fc51106de361f9fd1d1a5bf95c98b0ba09e Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 5 Sep 2024 15:01:04 -0500 Subject: [PATCH 26/33] fix unit tests --- tests/unittests/test_app_setting_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_app_setting_manager.py b/tests/unittests/test_app_setting_manager.py index e1646f9b8..88f1e3494 100644 --- a/tests/unittests/test_app_setting_manager.py +++ b/tests/unittests/test_app_setting_manager.py @@ -12,7 +12,7 @@ PYTHON_THREADPOOL_THREAD_COUNT, ) from azure_functions_worker.utils.app_setting_manager import get_python_appsetting_state -from azure_functions_worker.utils.config_manager import clear_config +from azure_functions_worker.utils.config_manager import clear_config, read_config SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) @@ -73,6 +73,7 @@ def setUpClass(cls): cls._patch_environ.start() super().setUpClass() clear_config() + read_config("") @classmethod def tearDownClass(cls): From bf8e509663f87006accf4997a2556d82edf85efa Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 16 Sep 2024 11:39:30 -0500 Subject: [PATCH 27/33] initial refactoring to config manager class --- .../file_accessor_factory.py | 4 +- .../file_accessor_unix.py | 4 +- .../shared_memory_manager.py | 4 +- azure_functions_worker/dispatcher.py | 33 ++- azure_functions_worker/http_v2.py | 4 +- azure_functions_worker/loader.py | 4 +- .../utils/app_setting_manager.py | 4 +- azure_functions_worker/utils/common.py | 2 +- .../utils/config_manager.py | 246 +++++++++--------- azure_functions_worker/utils/dependency.py | 22 +- azure_functions_worker/utils/wrappers.py | 10 +- .../test_dependency_isolation_functions.py | 10 +- tests/endtoend/test_durable_functions.py | 6 +- tests/endtoend/test_warmup_functions.py | 10 +- tests/unittests/test_app_setting_manager.py | 6 +- tests/unittests/test_dispatcher.py | 2 +- tests/unittests/test_extension.py | 4 +- tests/unittests/test_shared_memory_manager.py | 7 +- tests/unittests/test_utilities.py | 4 +- tests/unittests/test_utilities_dependency.py | 4 +- tests/utils/testutils.py | 26 +- 21 files changed, 206 insertions(+), 210 deletions(-) diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py index 6b22847fd..25c24916c 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_factory.py @@ -5,7 +5,7 @@ import sys from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED -from ...utils.config_manager import is_envvar_true +from ...utils.config_manager import config_manager from .file_accessor import DummyFileAccessor from .file_accessor_unix import FileAccessorUnix from .file_accessor_windows import FileAccessorWindows @@ -18,7 +18,7 @@ class FileAccessorFactory: """ @staticmethod def create_file_accessor(): - if sys.platform == "darwin" and not is_envvar_true( + if sys.platform == "darwin" and not config_manager.is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED): return DummyFileAccessor() elif os.name == 'nt': diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py index 15c4e3dd6..95d4919d4 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/file_accessor_unix.py @@ -9,7 +9,7 @@ from azure_functions_worker import constants from ...logging import logger -from ...utils.config_manager import get_app_setting +from ...utils.config_manager import config_manager from .file_accessor import FileAccessor from .shared_memory_constants import SharedMemoryConstants as consts from .shared_memory_exception import SharedMemoryException @@ -95,7 +95,7 @@ def _get_allowed_mem_map_dirs(self) -> List[str]: Otherwise, the default value will be used. """ setting = constants.UNIX_SHARED_MEMORY_DIRECTORIES - allowed_mem_map_dirs_str = get_app_setting(setting) + allowed_mem_map_dirs_str = config_manager.get_app_setting(setting) if allowed_mem_map_dirs_str is None: allowed_mem_map_dirs = consts.UNIX_TEMP_DIRS logger.info( diff --git a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py index 869295de9..25cee6218 100644 --- a/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py +++ b/azure_functions_worker/bindings/shared_memory_data_transfer/shared_memory_manager.py @@ -6,7 +6,7 @@ from ...constants import FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED from ...logging import logger -from ...utils.config_manager import is_envvar_true +from ...utils.config_manager import config_manager from ..datumdef import Datum from .file_accessor_factory import FileAccessorFactory from .shared_memory_constants import SharedMemoryConstants as consts @@ -55,7 +55,7 @@ def is_enabled(self) -> bool: Whether supported types should be transferred between functions host and the worker using shared memory. """ - return is_envvar_true( + return config_manager.is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) def is_supported(self, datum: Datum) -> bool: diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 1d023326f..65f901afa 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -61,8 +61,7 @@ ) from .utils.app_setting_manager import get_python_appsetting_state from .utils.common import validate_script_file_name -from .utils.config_manager import (clear_config, read_config, - is_envvar_true, get_app_setting) +from .utils.config_manager import config_manager from .utils.dependency import DependencyManager from .utils.tracing import marshall_exception_trace from .utils.wrappers import disable_feature_by @@ -209,7 +208,7 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression log_level = ( logging.INFO - if not is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING) + if not config_manager.is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING) else logging.DEBUG ) @@ -336,10 +335,10 @@ def initialize_azure_monitor(self): # Connection string can be explicitly specified in Appsetting # If not set, defaults to env var # APPLICATIONINSIGHTS_CONNECTION_STRING - connection_string=get_app_setting( + connection_string=config_manager.get_app_setting( setting=APPLICATIONINSIGHTS_CONNECTION_STRING ), - logger_name=get_app_setting( + logger_name=config_manager.get_app_setting( setting=PYTHON_AZURE_MONITOR_LOGGER_NAME, default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT, ), @@ -376,7 +375,7 @@ async def _handle__worker_init_request(self, request): self._sync_call_tp = self._create_sync_call_tp( self._get_sync_tp_max_workers() ) - read_config( + config_manager.read_config( os.path.join(worker_init_request.function_app_directory, "az-config.json") ) logger.info( @@ -406,7 +405,7 @@ async def _handle__worker_init_request(self, request): constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE, constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } - if get_app_setting( + if config_manager.get_app_setting( setting=PYTHON_ENABLE_OPENTELEMETRY, default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, ): @@ -425,7 +424,7 @@ async def _handle__worker_init_request(self, request): # dictionary which will be later used in the invocation request bindings.load_binding_registry() - if is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): + if config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( worker_init_request.function_app_directory, @@ -466,7 +465,7 @@ def load_function_metadata(self, function_app_directory, caller_info): directory and save the results in function_metadata_result or function_metadata_exception in case of an exception. """ - script_file_name = get_app_setting( + script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", ) @@ -494,7 +493,7 @@ async def _handle__functions_metadata_request(self, request): metadata_request = request.functions_metadata_request function_app_directory = metadata_request.function_app_directory - script_file_name = get_app_setting( + script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", ) @@ -506,7 +505,7 @@ async def _handle__functions_metadata_request(self, request): function_path, ) - if not is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): + if not config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( function_app_directory, caller_info="functions_metadata_request" @@ -819,11 +818,11 @@ async def _handle__function_environment_reload_request(self, request): # Reload environment variables os.environ.clear() - clear_config() + config_manager.clear_config() env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] - read_config( + config_manager.read_config( os.path.join( func_env_reload_request.function_app_directory, "az-config.json" ) @@ -835,7 +834,7 @@ async def _handle__function_environment_reload_request(self, request): self._get_sync_tp_max_workers() ) - if is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING): + if config_manager.is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING): root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) @@ -847,7 +846,7 @@ async def _handle__function_environment_reload_request(self, request): bindings.load_binding_registry() capabilities = {} - if get_app_setting( + if config_manager.get_app_setting( setting=PYTHON_ENABLE_OPENTELEMETRY, default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, ): @@ -856,7 +855,7 @@ async def _handle__function_environment_reload_request(self, request): if self._azure_monitor_available: capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = _TRUE - if is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): + if config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( directory, caller_info="environment_reload_request" @@ -1057,7 +1056,7 @@ def tp_max_workers_validator(value: str) -> bool: else f"{PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT}" ) - max_workers = get_app_setting( + max_workers = config_manager.get_app_setting( setting=PYTHON_THREADPOOL_THREAD_COUNT, default_value=default_value, validator=tp_max_workers_validator, diff --git a/azure_functions_worker/http_v2.py b/azure_functions_worker/http_v2.py index 74b71a472..dddfd4e5c 100644 --- a/azure_functions_worker/http_v2.py +++ b/azure_functions_worker/http_v2.py @@ -14,7 +14,7 @@ X_MS_INVOCATION_ID, ) from azure_functions_worker.logging import logger -from azure_functions_worker.utils.config_manager import is_envvar_false +from azure_functions_worker.utils.config_manager import config_manager # Http V2 Exceptions @@ -280,7 +280,7 @@ def ext_base(cls): @classmethod def _check_http_v2_enabled(cls): if sys.version_info.minor < BASE_EXT_SUPPORTED_PY_MINOR_VERSION or \ - is_envvar_false(PYTHON_ENABLE_INIT_INDEXING): + config_manager.is_envvar_false(PYTHON_ENABLE_INIT_INDEXING): return False import azurefunctions.extensions.base as ext_base diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 268263d00..4d013a3bf 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -26,7 +26,7 @@ RETRY_POLICY, ) from .logging import logger -from .utils.config_manager import get_app_setting +from .utils.config_manager import config_manager from .utils.wrappers import attach_message_to_exception _AZURE_NAMESPACE = '__app__' @@ -255,7 +255,7 @@ def index_function_app(function_path: str): f"level function app instances are defined.") if not app: - script_file_name = get_app_setting( + script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}') raise ValueError("Could not find top level function app instances in " diff --git a/azure_functions_worker/utils/app_setting_manager.py b/azure_functions_worker/utils/app_setting_manager.py index b71297820..9dcdb7dbc 100644 --- a/azure_functions_worker/utils/app_setting_manager.py +++ b/azure_functions_worker/utils/app_setting_manager.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import sys -from .config_manager import get_config +from .config_manager import config_manager from ..constants import ( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, PYTHON_ENABLE_DEBUG_LOGGING, @@ -19,7 +19,7 @@ def get_python_appsetting_state(): - current_vars = get_config() + current_vars = config_manager.get_config() python_specific_settings = [ PYTHON_ROLLBACK_CWD_PATH, PYTHON_THREADPOOL_THREAD_COUNT, diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index 94f5387bd..e0410b62b 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -4,7 +4,7 @@ import re import sys from types import ModuleType -import azure_functions_worker.utils.config_manager as config_manager +from azure_functions_worker.utils.config_manager import config_manager from azure_functions_worker.constants import ( CUSTOMER_PACKAGES_PATH, diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index 11b78962f..e67e6fcbd 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -4,131 +4,127 @@ import json from typing import Optional, Callable, Dict -# Initialize to None -config_data: Optional[Dict[str, str]] = None - - -def read_config(function_path: str): - global config_data - if config_data is None: - config_data = {} - try: - with open(function_path, "r") as stream: - # loads the entire json file - full_config_data = json.load(stream) - # gets the python section of the json file - config_data = full_config_data.get("PYTHON") - except FileNotFoundError: - pass - # updates the config dictionary with the environment variables - # this prioritizes set env variables over the config file - env_copy = os.environ - for k, v in env_copy.items(): - config_data.update({k.upper(): v}) - - -def config_exists() -> bool: - global config_data - if config_data is None: - read_config("") - return config_data is not None - - -def get_config() -> dict: - return config_data - - -def is_true_like(setting: str) -> bool: - if setting is None: - return False - - return setting.lower().strip() in {"1", "true", "t", "yes", "y"} - - -def is_false_like(setting: str) -> bool: - if setting is None: - return False - - return setting.lower().strip() in {"0", "false", "f", "no", "n"} - - -def is_envvar_true(key: str) -> bool: - key_upper = key.upper() - # special case for PYTHON_ENABLE_DEBUG_LOGGING - # This is read by the host and must be set in os.environ - if key_upper == "PYTHON_ENABLE_DEBUG_LOGGING": - val = os.getenv(key_upper) - return is_true_like(val) - if config_exists() and not config_data.get(key_upper): - return False - return is_true_like(config_data.get(key_upper)) - - -def is_envvar_false(key: str) -> bool: - key_upper = key.upper() - if config_exists() and not config_data.get(key_upper): - return False - return is_false_like(config_data.get(key_upper)) - - -def get_app_setting( - setting: str, - default_value: Optional[str] = None, - validator: Optional[Callable[[str], bool]] = None, -) -> Optional[str]: - """Returns the application setting from environment variable. - - Parameters - ---------- - setting: str - The name of the application setting (e.g. FUNCTIONS_RUNTIME_VERSION) - - default_value: Optional[str] - The expected return value when the application setting is not found, - or the app setting does not pass the validator. - - validator: Optional[Callable[[str], bool]] - A function accepts the app setting value and should return True when - the app setting value is acceptable. - - Returns - ------- - Optional[str] - A string value that is set in the application setting +class ConfigManager(object): """ - setting_upper = setting.upper() - if config_exists() and config_data.get(setting_upper) is not None: - # Setting exists, check with validator - app_setting_value = config_data.get(setting_upper) - - # If there's no validator, return the app setting value directly - if validator is None: - return app_setting_value - - # If the app setting is set with a validator, - # On True, should return the app setting value - # On False, should return the default value - if validator(app_setting_value): - return app_setting_value - - # Setting is not configured or validator is false - # Return default value - return default_value - - -def set_env_var(setting: str, value: str): - global config_data - config_data[setting] = value - - -def del_env_var(setting: str): - global config_data - config_data.pop(setting, None) - + // TODO: docs here + """ + def __init__(self): + """ + // TODO: docs here + """ + self.config_data: Optional[Dict[str, str]] = None + + def read_config(self, function_path: str): + if self.config_data is None: + self.config_data = {} + try: + with open(function_path, "r") as stream: + # loads the entire json file + full_config_data = json.load(stream) + # gets the python section of the json file + self.config_data = full_config_data.get("PYTHON") + except FileNotFoundError: + pass -def clear_config(): - global config_data - if config_data is not None: - config_data.clear() - config_data = None + # updates the config dictionary with the environment variables + # this prioritizes set env variables over the config file + env_copy = os.environ + for k, v in env_copy.items(): + self.config_data.update({k.upper(): v}) + + def config_exists(self) -> bool: + if self.config_data is None: + self.read_config("") + return self.config_data is not None + + def get_config(self) -> dict: + return self.config_data + + def is_true_like(self, setting: str) -> bool: + if setting is None: + return False + + return setting.lower().strip() in {"1", "true", "t", "yes", "y"} + + def is_false_like(self, setting: str) -> bool: + if setting is None: + return False + + return setting.lower().strip() in {"0", "false", "f", "no", "n"} + + def is_envvar_true(self, key: str) -> bool: + key_upper = key.upper() + # special case for PYTHON_ENABLE_DEBUG_LOGGING + # This is read by the host and must be set in os.environ + if key_upper == "PYTHON_ENABLE_DEBUG_LOGGING": + val = os.getenv(key_upper) + return self.is_true_like(val) + if self.config_exists() and not self.config_data.get(key_upper): + return False + return self.is_true_like(self.config_data.get(key_upper)) + + def is_envvar_false(self, key: str) -> bool: + key_upper = key.upper() + if self.config_exists() and not self.config_data.get(key_upper): + return False + return self.is_false_like(self.config_data.get(key_upper)) + + def get_app_setting( + self, + setting: str, + default_value: Optional[str] = None, + validator: Optional[Callable[[str], bool]] = None, + ) -> Optional[str]: + """Returns the application setting from environment variable. + + Parameters + ---------- + setting: str + The name of the application setting (e.g. FUNCTIONS_RUNTIME_VERSION) + + default_value: Optional[str] + The expected return value when the application setting is not found, + or the app setting does not pass the validator. + + validator: Optional[Callable[[str], bool]] + A function accepts the app setting value and should return True when + the app setting value is acceptable. + + Returns + ------- + Optional[str] + A string value that is set in the application setting + """ + setting_upper = setting.upper() + if self.config_exists() and self.config_data.get(setting_upper) is not None: + # Setting exists, check with validator + app_setting_value = self.config_data.get(setting_upper) + + # If there's no validator, return the app setting value directly + if validator is None: + return app_setting_value + + # If the app setting is set with a validator, + # On True, should return the app setting value + # On False, should return the default value + if validator(app_setting_value): + return app_setting_value + + # Setting is not configured or validator is false + # Return default value + return default_value + + def set_env_var(self, setting: str, value: str): + self.config_data[setting] = value + + def del_env_var(self, setting: str): + self.config_data.pop(setting, None) + + def clear_config(self): + if self.config_data is not None: + self.config_data.clear() + self.config_data = None + + +config_manager = ConfigManager() diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index e8f58a19c..5e36e858a 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -8,9 +8,7 @@ from types import ModuleType from typing import List, Optional -from azure_functions_worker.utils.config_manager import (is_true_like, - is_envvar_true, - get_app_setting) +from azure_functions_worker.utils.config_manager import config_manager from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, CONTAINER_NAME, @@ -74,7 +72,7 @@ def initialize(cls): @classmethod def is_in_linux_consumption(cls): - return get_app_setting(CONTAINER_NAME) is not None + return config_manager.get_app_setting(CONTAINER_NAME) is not None @classmethod def should_load_cx_dependencies(cls): @@ -87,7 +85,7 @@ def should_load_cx_dependencies(cls): (OOM, timeouts etc) and env reload request is not called. """ return not (DependencyManager.is_in_linux_consumption() - and is_envvar_true("WEBSITE_PLACEHOLDER_MODE")) + and config_manager.is_envvar_true("WEBSITE_PLACEHOLDER_MODE")) @classmethod @enable_feature_by( @@ -146,7 +144,8 @@ def prioritize_customer_dependencies(cls, cx_working_dir=None): if not working_directory: working_directory = cls.cx_working_dir if not working_directory: - working_directory = get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT, '') + working_directory = config_manager.get_app_setting( + AZURE_WEBJOBS_SCRIPT_ROOT, '') # Try to get the latest customer's dependency path cx_deps_path: str = cls._get_cx_deps_path() @@ -159,7 +158,7 @@ def prioritize_customer_dependencies(cls, cx_working_dir=None): 'working_directory: %s, Linux Consumption: %s, Placeholder: %s', cls.worker_deps_path, cx_deps_path, working_directory, DependencyManager.is_in_linux_consumption(), - is_envvar_true("WEBSITE_PLACEHOLDER_MODE")) + config_manager.is_envvar_true("WEBSITE_PLACEHOLDER_MODE")) cls._remove_from_sys_path(cls.worker_deps_path) cls._add_to_sys_path(cls.cx_deps_path, True) @@ -194,7 +193,7 @@ def reload_customer_libraries(cls, cx_working_dir: str): cx_working_dir: str The path which contains customer's project file (e.g. wwwroot). """ - use_new_env = get_app_setting(PYTHON_ISOLATE_WORKER_DEPENDENCIES) + use_new_env = config_manager.get_app_setting(PYTHON_ISOLATE_WORKER_DEPENDENCIES) if use_new_env is None: use_new = ( PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 if @@ -202,7 +201,7 @@ def reload_customer_libraries(cls, cx_working_dir: str): PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT ) else: - use_new = is_true_like(use_new_env) + use_new = config_manager.is_true_like(use_new_env) if use_new: cls.prioritize_customer_dependencies(cx_working_dir) @@ -315,7 +314,8 @@ def _get_cx_deps_path() -> str: Linux Dedicated/Premium: path to customer's site pacakges Linux Consumption: empty string """ - prefix: Optional[str] = get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT) + prefix: Optional[str] = config_manager.get_app_setting( + AZURE_WEBJOBS_SCRIPT_ROOT) cx_paths: List[str] = [ p for p in sys.path if prefix and p.startswith(prefix) and ('site-packages' in p) @@ -334,7 +334,7 @@ def _get_cx_working_dir() -> str: Linux Dedicated/Premium: AzureWebJobsScriptRoot env variable Linux Consumption: empty string """ - return get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT, '') + return config_manager.get_app_setting(AZURE_WEBJOBS_SCRIPT_ROOT, '') @staticmethod def _get_worker_deps_path() -> str: diff --git a/azure_functions_worker/utils/wrappers.py b/azure_functions_worker/utils/wrappers.py index 94bf97959..979bb7c29 100644 --- a/azure_functions_worker/utils/wrappers.py +++ b/azure_functions_worker/utils/wrappers.py @@ -4,7 +4,7 @@ from typing import Any, Callable from ..logging import error_logger, logger -from .config_manager import is_envvar_false, is_envvar_true +from .config_manager import config_manager from .tracing import extend_exception_message @@ -13,9 +13,9 @@ def enable_feature_by(flag: str, flag_default: bool = False) -> Callable: def decorate(func): def call(*args, **kwargs): - if is_envvar_true(flag): + if config_manager.is_envvar_true(flag): return func(*args, **kwargs) - if flag_default and not is_envvar_false(flag): + if flag_default and not config_manager.is_envvar_false(flag): return func(*args, **kwargs) return default return call @@ -27,9 +27,9 @@ def disable_feature_by(flag: str, flag_default: bool = False) -> Callable: def decorate(func): def call(*args, **kwargs): - if is_envvar_true(flag): + if config_manager.is_envvar_true(flag): return default - if flag_default and not is_envvar_false(flag): + if flag_default and not config_manager.is_envvar_false(flag): return default return func(*args, **kwargs) return call diff --git a/tests/endtoend/test_dependency_isolation_functions.py b/tests/endtoend/test_dependency_isolation_functions.py index f4ce902a4..d94852a6d 100644 --- a/tests/endtoend/test_dependency_isolation_functions.py +++ b/tests/endtoend/test_dependency_isolation_functions.py @@ -14,13 +14,13 @@ PYAZURE_INTEGRATION_TEST, ) -from azure_functions_worker.utils.config_manager import is_envvar_true +from azure_functions_worker.utils.config_manager import config_manager REQUEST_TIMEOUT_SEC = 5 -@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), +@skipIf(config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) + or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST), 'Docker tests do not work with dependency isolation ') class TestGRPCandProtobufDependencyIsolationOnDedicated( testutils.WebHostTestCase): @@ -95,7 +95,7 @@ def test_working_directory_resolution(self): os.path.join(dir, 'dependency_isolation_functions').lower() ) - @skipIf(is_envvar_true(PYAZURE_INTEGRATION_TEST), + @skipIf(config_manager.is_envvar_true(PYAZURE_INTEGRATION_TEST), 'Integration test expects dependencies derived from core ' 'tools folder') def test_paths_resolution(self): @@ -121,7 +121,7 @@ def test_paths_resolution(self): ).lower() ) - @skipIf(is_envvar_true('USETESTPYTHONSDK'), + @skipIf(config_manager.is_envvar_true('USETESTPYTHONSDK'), 'Running tests using an editable azure-functions package.') def test_loading_libraries_from_customers_package(self): """Since the Python now loaded the customer's dependencies, the diff --git a/tests/endtoend/test_durable_functions.py b/tests/endtoend/test_durable_functions.py index 94f58285c..d792b04cb 100644 --- a/tests/endtoend/test_durable_functions.py +++ b/tests/endtoend/test_durable_functions.py @@ -9,11 +9,11 @@ from tests.utils import testutils from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -from azure_functions_worker.utils.config_manager import is_envvar_true +from azure_functions_worker.utils.config_manager import config_manager -@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), +@skipIf(config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) + or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST), "Docker tests cannot retrieve port needed for a webhook") class TestDurableFunctions(testutils.WebHostTestCase): diff --git a/tests/endtoend/test_warmup_functions.py b/tests/endtoend/test_warmup_functions.py index 93bca3ecd..b1cbb2238 100644 --- a/tests/endtoend/test_warmup_functions.py +++ b/tests/endtoend/test_warmup_functions.py @@ -7,11 +7,11 @@ from tests.utils import testutils from tests.utils.constants import CONSUMPTION_DOCKER_TEST, DEDICATED_DOCKER_TEST -from azure_functions_worker.utils.config_manager import is_envvar_true +from azure_functions_worker.utils.config_manager import config_manager -@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), +@skipIf(config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) + or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST), "Docker tests cannot call admin functions") class TestWarmupFunctions(testutils.WebHostTestCase): """Test the Warmup Trigger in the local webhost. @@ -36,8 +36,8 @@ def check_log_warmup(self, host_out: typing.List[str]): self.assertEqual(host_out.count("Function App instance is warm"), 1) -@skipIf(is_envvar_true(DEDICATED_DOCKER_TEST) - or is_envvar_true(CONSUMPTION_DOCKER_TEST), +@skipIf(config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) + or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST), "Docker tests cannot call admin functions") class TestWarmupFunctionsStein(TestWarmupFunctions): diff --git a/tests/unittests/test_app_setting_manager.py b/tests/unittests/test_app_setting_manager.py index 88f1e3494..f49cccaa9 100644 --- a/tests/unittests/test_app_setting_manager.py +++ b/tests/unittests/test_app_setting_manager.py @@ -12,7 +12,7 @@ PYTHON_THREADPOOL_THREAD_COUNT, ) from azure_functions_worker.utils.app_setting_manager import get_python_appsetting_state -from azure_functions_worker.utils.config_manager import clear_config, read_config +from azure_functions_worker.utils.config_manager import config_manager SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) @@ -72,8 +72,8 @@ def setUpClass(cls): cls._patch_environ = patch.dict('os.environ', os_environ) cls._patch_environ.start() super().setUpClass() - clear_config() - read_config("") + config_manager.clear_config() + config_manager.read_config("") @classmethod def tearDownClass(cls): diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index d1bf50024..5a68bc4db 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -23,7 +23,7 @@ PYTHON_THREADPOOL_THREAD_COUNT_MIN, HTTP_URI, REQUIRES_ROUTE_PARAMETERS, ) from azure_functions_worker.dispatcher import Dispatcher, ContextEnabledTask -from azure_functions_worker.utils import config_manager +from azure_functions_worker.utils.config_manager import config_manager from azure_functions_worker.version import VERSION SysVersionInfo = col.namedtuple("VersionInfo", ["major", "minor", "micro", diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index b10ee184f..8bf016557 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -24,7 +24,7 @@ ExtensionManager, ) from azure_functions_worker.utils.common import get_sdk_from_sys_path -from azure_functions_worker.utils.config_manager import clear_config +from azure_functions_worker.utils.config_manager import config_manager class MockContext: @@ -76,7 +76,7 @@ def setUp(self): # Set feature flag to on os.environ[PYTHON_ENABLE_WORKER_EXTENSIONS] = 'true' - clear_config() + config_manager.clear_config() def tearDown(self) -> None: os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) diff --git a/tests/unittests/test_shared_memory_manager.py b/tests/unittests/test_shared_memory_manager.py index 80d3e683d..aaa366feb 100644 --- a/tests/unittests/test_shared_memory_manager.py +++ b/tests/unittests/test_shared_memory_manager.py @@ -9,7 +9,6 @@ from unittest.mock import patch from azure.functions import meta as bind_meta -from azure_functions_worker.utils import config_manager from tests.utils import testutils from azure_functions_worker.bindings.shared_memory_data_transfer import ( @@ -21,7 +20,7 @@ from azure_functions_worker.constants import ( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, ) -from azure_functions_worker.utils.config_manager import is_envvar_true +from azure_functions_worker.utils.config_manager import config_manager @skipIf(sys.platform == 'darwin', 'MacOS M1 machines do not correctly test the' @@ -54,7 +53,7 @@ def test_is_enabled(self): """ # Make sure shared memory data transfer is enabled - was_shmem_env_true = is_envvar_true( + was_shmem_env_true = config_manager.is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) os.environ.update( {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) @@ -71,7 +70,7 @@ def test_is_disabled(self): disabled. """ # Make sure shared memory data transfer is disabled - was_shmem_env_true = is_envvar_true( + was_shmem_env_true = config_manager.is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) config_manager.clear_config() os.environ.update( diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 34dc9f996..a1a23677b 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -8,7 +8,8 @@ from unittest.mock import patch from azure_functions_worker.constants import PYTHON_EXTENSIONS_RELOAD_FUNCTIONS -from azure_functions_worker.utils import common, config_manager, wrappers +from azure_functions_worker.utils import common, wrappers +from azure_functions_worker.utils.config_manager import config_manager TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" TEST_FEATURE_FLAG = "APP_SETTING_FEATURE_FLAG" @@ -82,6 +83,7 @@ def setUp(self): self.mock_environ.start() self.mock_sys_module.start() self.mock_sys_path.start() + config_manager.clear_config() def tearDown(self): self.mock_sys_path.stop() diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index a88203ecf..fe4ac57f2 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -7,7 +7,7 @@ from unittest.mock import patch from azure_functions_worker.utils.dependency import DependencyManager -from azure_functions_worker.utils.config_manager import clear_config +from azure_functions_worker.utils.config_manager import config_manager from tests.utils import testutils @@ -38,7 +38,7 @@ def setUp(self): self._patch_sys_path.start() self._patch_importer_cache.start() self._patch_modules.start() - clear_config() + config_manager.clear_config() def tearDown(self): self._patch_environ.stop() diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index f464099b5..3db2320f1 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -61,8 +61,7 @@ FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, UNIX_SHARED_MEMORY_DIRECTORIES, ) -from azure_functions_worker.utils.config_manager import (is_envvar_true, - get_app_setting) +from azure_functions_worker.utils.config_manager import config_manager TESTS_ROOT = PROJECT_ROOT / 'tests' E2E_TESTS_FOLDER = pathlib.Path('endtoend') @@ -142,8 +141,8 @@ class AsyncTestCase(unittest.TestCase, metaclass=AsyncTestCaseMeta): class WebHostTestCaseMeta(type(unittest.TestCase)): def __new__(mcls, name, bases, dct): - if is_envvar_true(DEDICATED_DOCKER_TEST) \ - or is_envvar_true(CONSUMPTION_DOCKER_TEST): + if config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) \ + or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST): return super().__new__(mcls, name, bases, dct) for attrname, attr in dct.items(): @@ -157,7 +156,8 @@ def wrapper(self, *args, __meth__=test_case, __check_log__=check_log_case, **kwargs): if (__check_log__ is not None and callable(__check_log__) - and not is_envvar_true(PYAZURE_WEBHOST_DEBUG)): + and not config_manager.is_envvar_true( + PYAZURE_WEBHOST_DEBUG)): # Check logging output for unit test scenarios result = self._run_test(__meth__, *args, **kwargs) @@ -217,9 +217,9 @@ def docker_tests_enabled(self) -> (bool, str): CONSUMPTION_DOCKER_TEST or DEDICATED_DOCKER_TEST is enabled else returns False """ - if is_envvar_true(CONSUMPTION_DOCKER_TEST): + if config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST): return True, CONSUMPTION_DOCKER_TEST - elif is_envvar_true(DEDICATED_DOCKER_TEST): + elif config_manager.is_envvar_true(DEDICATED_DOCKER_TEST): return True, DEDICATED_DOCKER_TEST else: return False, None @@ -231,7 +231,7 @@ def setUpClass(cls): docker_tests_enabled, sku = cls.docker_tests_enabled() - cls.host_stdout = None if is_envvar_true(PYAZURE_WEBHOST_DEBUG) \ + cls.host_stdout = None if config_manager.is_envvar_true(PYAZURE_WEBHOST_DEBUG) \ else tempfile.NamedTemporaryFile('w+t') try: @@ -272,7 +272,7 @@ def tearDownClass(cls): cls.webhost = None if cls.host_stdout is not None: - if is_envvar_true(ARCHIVE_WEBHOST_LOGS): + if config_manager.is_envvar_true(ARCHIVE_WEBHOST_LOGS): cls.host_stdout.seek(0) content = cls.host_stdout.read() if content is not None and len(content) > 0: @@ -327,7 +327,7 @@ class SharedMemoryTestCase(unittest.TestCase): """ def setUp(self): - self.was_shmem_env_true = is_envvar_true( + self.was_shmem_env_true = config_manager.is_envvar_true( FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED) os.environ.update( {FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED: '1'}) @@ -336,7 +336,7 @@ def setUp(self): if os_name == 'Darwin': # If an existing AppSetting is specified, save it so it can be # restored later - self.was_shmem_dirs = get_app_setting( + self.was_shmem_dirs = config_manager.get_app_setting( UNIX_SHARED_MEMORY_DIRECTORIES ) self._setUpDarwin() @@ -917,7 +917,7 @@ def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None): # In E2E Integration mode, we should use the core tools worker # from the latest artifact instead of the azure_functions_worker module - if is_envvar_true(PYAZURE_INTEGRATION_TEST): + if config_manager.is_envvar_true(PYAZURE_INTEGRATION_TEST): extra_env.pop('languageWorkers:python:workerDirectory') if testconfig and 'azure' in testconfig: @@ -967,7 +967,7 @@ def popen_webhost(*, stdout, stderr, script_root=FUNCS_PATH, port=None): def start_webhost(*, script_dir=None, stdout=None): script_root = TESTS_ROOT / script_dir if script_dir else FUNCS_PATH if stdout is None: - if is_envvar_true(PYAZURE_WEBHOST_DEBUG): + if config_manager.is_envvar_true(PYAZURE_WEBHOST_DEBUG): stdout = sys.stdout else: stdout = subprocess.DEVNULL From bcf434b98946eca1ed4c96b98f7d9145f0495e6a Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 16 Sep 2024 13:37:19 -0500 Subject: [PATCH 28/33] reference bug fix --- tests/unittests/test_opentelemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_opentelemetry.py b/tests/unittests/test_opentelemetry.py index 58121fe68..c528b1777 100644 --- a/tests/unittests/test_opentelemetry.py +++ b/tests/unittests/test_opentelemetry.py @@ -7,7 +7,7 @@ from tests.utils import testutils from azure_functions_worker import protos -from azure_functions_worker.utils import config_manager +from azure_functions_worker.utils.config_manager import config_manager class TestOpenTelemetry(unittest.TestCase): From 244ba610e8a68e3dcd523be8665aa69ba0be93f0 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 17 Sep 2024 12:17:02 -0500 Subject: [PATCH 29/33] refactoring for env var reading --- azure_functions_worker/dispatcher.py | 5 +++-- .../utils/config_manager.py | 19 +++++++++++++++---- tests/utils/testutils.py | 1 + 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 65f901afa..110c98b83 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -115,6 +115,7 @@ def __init__( self._context_api = None self._trace_context_propagator = None + config_manager.read_environment_variables() # We allow the customer to change synchronous thread pool max worker # count by setting the PYTHON_THREADPOOL_THREAD_COUNT app setting. # For 3.[6|7|8] The default value is 1. @@ -375,7 +376,7 @@ async def _handle__worker_init_request(self, request): self._sync_call_tp = self._create_sync_call_tp( self._get_sync_tp_max_workers() ) - config_manager.read_config( + config_manager.set_config( os.path.join(worker_init_request.function_app_directory, "az-config.json") ) logger.info( @@ -822,7 +823,7 @@ async def _handle__function_environment_reload_request(self, request): env_vars = func_env_reload_request.environment_variables for var in env_vars: os.environ[var] = env_vars[var] - config_manager.read_config( + config_manager.set_config( os.path.join( func_env_reload_request.function_app_directory, "az-config.json" ) diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index e67e6fcbd..c16706cc2 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -15,6 +15,13 @@ def __init__(self): """ self.config_data: Optional[Dict[str, str]] = None + def read_environment_variables(self): + if self.config_data is None: + self.config_data = {} + env_copy = os.environ + for k, v in env_copy.items(): + self.config_data.update({k.upper(): v}) + def read_config(self, function_path: str): if self.config_data is None: self.config_data = {} @@ -29,13 +36,17 @@ def read_config(self, function_path: str): # updates the config dictionary with the environment variables # this prioritizes set env variables over the config file - env_copy = os.environ - for k, v in env_copy.items(): - self.config_data.update({k.upper(): v}) + # env_copy = os.environ + # for k, v in env_copy.items(): + # self.config_data.update({k.upper(): v}) + + def set_config(self, function_path: str): + self.read_config(function_path) + self.read_environment_variables() def config_exists(self) -> bool: if self.config_data is None: - self.read_config("") + self.set_config("") return self.config_data is not None def get_config(self) -> dict: diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index 3db2320f1..d7773ca2d 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -141,6 +141,7 @@ class AsyncTestCase(unittest.TestCase, metaclass=AsyncTestCaseMeta): class WebHostTestCaseMeta(type(unittest.TestCase)): def __new__(mcls, name, bases, dct): + config_manager.read_environment_variables() if config_manager.is_envvar_true(DEDICATED_DOCKER_TEST) \ or config_manager.is_envvar_true(CONSUMPTION_DOCKER_TEST): return super().__new__(mcls, name, bases, dct) From 88ef536c3b4dc71a86980a5d489a448ff4180c47 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 17 Sep 2024 12:57:45 -0500 Subject: [PATCH 30/33] test fixes --- tests/unittests/test_app_setting_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittests/test_app_setting_manager.py b/tests/unittests/test_app_setting_manager.py index f49cccaa9..4d437001c 100644 --- a/tests/unittests/test_app_setting_manager.py +++ b/tests/unittests/test_app_setting_manager.py @@ -73,7 +73,7 @@ def setUpClass(cls): cls._patch_environ.start() super().setUpClass() config_manager.clear_config() - config_manager.read_config("") + config_manager.read_environment_variables() @classmethod def tearDownClass(cls): From af36a6e94f51188d8e6e98f278db43c9e4ce06b9 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 17 Sep 2024 14:39:13 -0500 Subject: [PATCH 31/33] formatting fixes --- azure_functions_worker/dispatcher.py | 534 ++++++++---------- .../utils/app_setting_manager.py | 37 +- .../utils/config_manager.py | 27 +- tests/unittests/test_app_setting_manager.py | 1 - tests/unittests/test_dispatcher.py | 8 - tests/unittests/test_extension.py | 1 - .../test_third_party_http_functions.py | 1 - tests/unittests/test_utilities.py | 1 - 8 files changed, 251 insertions(+), 359 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 110c98b83..753e6e543 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -30,7 +30,6 @@ PYTHON_ENABLE_DEBUG_LOGGING, PYTHON_ENABLE_INIT_INDEXING, PYTHON_ENABLE_OPENTELEMETRY, - PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, PYTHON_LANGUAGE_RUNTIME, PYTHON_ROLLBACK_CWD_PATH, PYTHON_SCRIPT_FILE_NAME, @@ -79,23 +78,17 @@ class DispatcherMeta(type): def current(mcls): disp = mcls.__current_dispatcher__ if disp is None: - raise RuntimeError("no currently running Dispatcher is found") + raise RuntimeError('no currently running Dispatcher is found') return disp class Dispatcher(metaclass=DispatcherMeta): _GRPC_STOP_RESPONSE = object() - def __init__( - self, - loop: BaseEventLoop, - host: str, - port: int, - worker_id: str, - request_id: str, - grpc_connect_timeout: float, - grpc_max_msg_len: int = -1, - ) -> None: + def __init__(self, loop: BaseEventLoop, host: str, port: int, + worker_id: str, request_id: str, + grpc_connect_timeout: float, + grpc_max_msg_len: int = -1) -> None: self._loop = loop self._host = host self._port = port @@ -115,14 +108,14 @@ def __init__( self._context_api = None self._trace_context_propagator = None - config_manager.read_environment_variables() # We allow the customer to change synchronous thread pool max worker # count by setting the PYTHON_THREADPOOL_THREAD_COUNT app setting. # For 3.[6|7|8] The default value is 1. # For 3.9, we don't set this value by default but we honor incoming # the app setting. - self._sync_call_tp: concurrent.futures.Executor = self._create_sync_call_tp( - self._get_sync_tp_max_workers() + config_manager.read_environment_variables() + self._sync_call_tp: concurrent.futures.Executor = ( + self._create_sync_call_tp(self._get_sync_tp_max_workers()) ) self._grpc_connect_timeout: float = grpc_connect_timeout @@ -131,49 +124,40 @@ def __init__( self._grpc_resp_queue: queue.Queue = queue.Queue() self._grpc_connected_fut = loop.create_future() self._grpc_thread: threading.Thread = threading.Thread( - name="grpc-thread", target=self.__poll_grpc - ) + name='grpc-thread', target=self.__poll_grpc) @staticmethod def get_worker_metadata(): return protos.WorkerMetadata( runtime_name=PYTHON_LANGUAGE_RUNTIME, - runtime_version=f"{sys.version_info.major}." f"{sys.version_info.minor}", + runtime_version=f"{sys.version_info.major}." + f"{sys.version_info.minor}", worker_version=VERSION, worker_bitness=platform.machine(), - custom_properties={}, - ) + custom_properties={}) def get_sync_tp_workers_set(self): """We don't know the exact value of the threadcount set for the Python - 3.9 scenarios (as we'll start passing only None by default), and we - need to get that information. - - Ref: concurrent.futures.thread.ThreadPoolExecutor.__init__._max_workers + 3.9 scenarios (as we'll start passing only None by default), and we + need to get that information. + Ref: concurrent.futures.thread.ThreadPoolExecutor.__init__._max_workers """ return self._sync_call_tp._max_workers @classmethod - async def connect( - cls, - host: str, - port: int, - worker_id: str, - request_id: str, - connect_timeout: float, - ): + async def connect(cls, host: str, port: int, worker_id: str, + request_id: str, connect_timeout: float): loop = asyncio.events.get_event_loop() disp = cls(loop, host, port, worker_id, request_id, connect_timeout) disp._grpc_thread.start() await disp._grpc_connected_fut - logger.info("Successfully opened gRPC channel to %s:%s ", host, port) + logger.info('Successfully opened gRPC channel to %s:%s ', host, port) return disp async def dispatch_forever(self): # sourcery skip: swap-if-expression if DispatcherMeta.__current_dispatcher__ is not None: - raise RuntimeError( - "there can be only one running dispatcher per " "process" - ) + raise RuntimeError('there can be only one running dispatcher per ' + 'process') self._old_task_factory = self._loop.get_task_factory() @@ -186,20 +170,17 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression self._grpc_resp_queue.put_nowait( protos.StreamingMessage( request_id=self.request_id, - start_stream=protos.StartStream(worker_id=self.worker_id), - ) - ) + start_stream=protos.StartStream( + worker_id=self.worker_id))) # In Python 3.11+, constructing a task has an optional context # parameter. Allow for this param to be passed to ContextEnabledTask self._loop.set_task_factory( lambda loop, coro, context=None: ContextEnabledTask( - coro, loop=loop, context=context - ) - ) + coro, loop=loop, context=context)) # Detach console logging before enabling GRPC channel logging - logger.info("Detaching console logging.") + logger.info('Detaching console logging.') disable_console_logging() # Attach gRPC logging to the root logger. Since gRPC channel is @@ -207,11 +188,8 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression logging_handler = AsyncLoggingHandler() root_logger = logging.getLogger() - log_level = ( - logging.INFO - if not config_manager.is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING) - else logging.DEBUG - ) + log_level = logging.INFO if not config_manager.is_envvar_true( + PYTHON_ENABLE_DEBUG_LOGGING) else logging.DEBUG root_logger.setLevel(log_level) root_logger.addHandler(logging_handler) @@ -244,7 +222,8 @@ def stop(self) -> None: self._stop_sync_call_tp() - def on_logging(self, record: logging.LogRecord, formatted_msg: str) -> None: + def on_logging(self, record: logging.LogRecord, + formatted_msg: str) -> None: if record.levelno >= logging.CRITICAL: log_level = protos.RpcLog.Critical elif record.levelno >= logging.ERROR: @@ -267,7 +246,7 @@ def on_logging(self, record: logging.LogRecord, formatted_msg: str) -> None: level=log_level, message=formatted_msg, category=record.name, - log_category=log_category, + log_category=log_category ) invocation_id = get_current_invocation_id() @@ -276,9 +255,8 @@ def on_logging(self, record: logging.LogRecord, formatted_msg: str) -> None: self._grpc_resp_queue.put_nowait( protos.StreamingMessage( - request_id=self.request_id, rpc_log=protos.RpcLog(**log) - ) - ) + request_id=self.request_id, + rpc_log=protos.RpcLog(**log))) @property def request_id(self) -> str: @@ -294,10 +272,8 @@ def _serialize_exception(exc: Exception): try: message = f"{type(exc).__name__}: {exc}" except Exception: - message = ( - "Unhandled exception in function. " - "Could not serialize original exception message." - ) + message = ('Unhandled exception in function. ' + 'Could not serialize original exception message.') try: stack_trace = marshall_exception_trace(exc) @@ -313,14 +289,16 @@ async def _dispatch_grpc_request(self, request): # Don't crash on unknown messages. Some of them can be ignored; # and if something goes really wrong the host can always just # kill the worker's process. - logger.error("unknown StreamingMessage content type %s", content_type) + logger.error('unknown StreamingMessage content type %s', + content_type) return resp = await request_handler(request) self._grpc_resp_queue.put_nowait(resp) def initialize_azure_monitor(self): - """Initializes OpenTelemetry and Azure monitor distro""" + """Initializes OpenTelemetry and Azure monitor distro + """ self.update_opentelemetry_status() try: from azure.monitor.opentelemetry import configure_azure_monitor @@ -341,17 +319,21 @@ def initialize_azure_monitor(self): ), logger_name=config_manager.get_app_setting( setting=PYTHON_AZURE_MONITOR_LOGGER_NAME, - default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT, + default_value=PYTHON_AZURE_MONITOR_LOGGER_NAME_DEFAULT ), ) self._azure_monitor_available = True logger.info("Successfully configured Azure monitor distro.") except ImportError: - logger.exception("Cannot import Azure Monitor distro.") + logger.exception( + "Cannot import Azure Monitor distro." + ) self._azure_monitor_available = False except Exception: - logger.exception("Error initializing Azure monitor distro.") + logger.exception( + "Error initializing Azure monitor distro." + ) self._azure_monitor_available = False def update_opentelemetry_status(self): @@ -367,15 +349,12 @@ def update_opentelemetry_status(self): self._trace_context_propagator = TraceContextTextMapPropagator() except ImportError: - logger.exception("Cannot import OpenTelemetry libraries.") + logger.exception( + "Cannot import OpenTelemetry libraries." + ) async def _handle__worker_init_request(self, request): worker_init_request = request.worker_init_request - # Apply PYTHON_THREADPOOL_THREAD_COUNT - self._stop_sync_call_tp() - self._sync_call_tp = self._create_sync_call_tp( - self._get_sync_tp_max_workers() - ) config_manager.set_config( os.path.join(worker_init_request.function_app_directory, "az-config.json") ) @@ -406,10 +385,7 @@ async def _handle__worker_init_request(self, request): constants.RPC_HTTP_TRIGGER_METADATA_REMOVED: _TRUE, constants.SHARED_MEMORY_DATA_TRANSFER: _TRUE, } - if config_manager.get_app_setting( - setting=PYTHON_ENABLE_OPENTELEMETRY, - default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, - ): + if config_manager.is_envvar_true(PYTHON_ENABLE_OPENTELEMETRY): self.initialize_azure_monitor() if self._azure_monitor_available: @@ -429,8 +405,7 @@ async def _handle__worker_init_request(self, request): try: self.load_function_metadata( worker_init_request.function_app_directory, - caller_info="worker_init_request", - ) + caller_info="worker_init_request") if HttpV2Registry.http_v2_enabled(): capabilities[HTTP_URI] = \ @@ -447,9 +422,8 @@ async def _handle__worker_init_request(self, request): worker_init_response=protos.WorkerInitResponse( capabilities=capabilities, worker_metadata=self.get_worker_metadata(), - result=protos.StatusResult(status=protos.StatusResult.Success), - ), - ) + result=protos.StatusResult( + status=protos.StatusResult.Success))) async def _handle__worker_status_request(self, request): # Logging is not necessary in this request since the response is used @@ -457,8 +431,7 @@ async def _handle__worker_status_request(self, request): # Having log here will reduce the responsiveness of the worker. return protos.StreamingMessage( request_id=request.request_id, - worker_status_response=protos.WorkerStatusResponse(), - ) + worker_status_response=protos.WorkerStatusResponse()) def load_function_metadata(self, function_app_directory, caller_info): """ @@ -468,27 +441,22 @@ def load_function_metadata(self, function_app_directory, caller_info): """ script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, - default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", - ) + default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}") logger.debug( - "Received load metadata request from %s, request ID %s, " - "script_file_name: %s", - caller_info, - self.request_id, - script_file_name, - ) + 'Received load metadata request from %s, request ID %s, ' + 'script_file_name: %s', + caller_info, self.request_id, script_file_name) validate_script_file_name(script_file_name) - function_path = os.path.join(function_app_directory, script_file_name) + function_path = os.path.join(function_app_directory, + script_file_name) # For V1, the function path will not exist and # return None. self._function_metadata_result = ( - (self.index_functions(function_path, function_app_directory)) - if os.path.exists(function_path) - else None - ) + self.index_functions(function_path, function_app_directory)) \ + if os.path.exists(function_path) else None async def _handle__functions_metadata_request(self, request): metadata_request = request.functions_metadata_request @@ -496,9 +464,9 @@ async def _handle__functions_metadata_request(self, request): script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, - default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}", - ) - function_path = os.path.join(function_app_directory, script_file_name) + default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}') + function_path = os.path.join(function_app_directory, + script_file_name) logger.info( "Received WorkerMetadataRequest, request ID %s, " "function_path: %s", @@ -509,8 +477,8 @@ async def _handle__functions_metadata_request(self, request): if not config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( - function_app_directory, caller_info="functions_metadata_request" - ) + function_app_directory, + caller_info="functions_metadata_request") except Exception as ex: self._function_metadata_exception = ex @@ -521,22 +489,18 @@ async def _handle__functions_metadata_request(self, request): result=protos.StatusResult( status=protos.StatusResult.Failure, exception=self._serialize_exception( - self._function_metadata_exception - ), - ) - ), - ) + self._function_metadata_exception)))) else: metadata_result = self._function_metadata_result return protos.StreamingMessage( request_id=request.request_id, function_metadata_response=protos.FunctionMetadataResponse( - use_default_metadata_indexing=False if metadata_result else True, + use_default_metadata_indexing=False if metadata_result else + True, function_metadata_results=metadata_result, - result=protos.StatusResult(status=protos.StatusResult.Success), - ), - ) + result=protos.StatusResult( + status=protos.StatusResult.Success))) async def _handle__function_load_request(self, request): func_request = request.function_load_request @@ -546,21 +510,17 @@ async def _handle__function_load_request(self, request): function_app_directory = function_metadata.directory logger.info( - "Received WorkerLoadRequest, request ID %s, function_id: %s," - "function_name: %s, function_app_directory : %s", - self.request_id, - function_id, - function_name, - function_app_directory, - ) + 'Received WorkerLoadRequest, request ID %s, function_id: %s,' + 'function_name: %s, function_app_directory : %s', + self.request_id, function_id, function_name, + function_app_directory) programming_model = "V2" try: if not self._functions.get_function(function_id): if function_metadata.properties.get( - METADATA_PROPERTIES_WORKER_INDEXED, False - ): + METADATA_PROPERTIES_WORKER_INDEXED, False): # This is for the second worker and above where the worker # indexing is enabled and load request is called without # calling the metadata request. In this case we index the @@ -568,8 +528,8 @@ async def _handle__function_load_request(self, request): try: self.load_function_metadata( - function_app_directory, caller_info="functions_load_request" - ) + function_app_directory, + caller_info="functions_load_request") except Exception as ex: self._function_metadata_exception = ex @@ -586,40 +546,36 @@ async def _handle__function_load_request(self, request): function_name, function_app_directory, func_request.metadata.script_file, - func_request.metadata.entry_point, - ) + func_request.metadata.entry_point) self._functions.add_function( - function_id, func, func_request.metadata - ) + function_id, func, func_request.metadata) try: ExtensionManager.function_load_extension( - function_name, func_request.metadata.directory + function_name, + func_request.metadata.directory ) except Exception as ex: logging.error("Failed to load extensions: ", ex) raise - logger.info( - "Successfully processed FunctionLoadRequest, " - "request ID: %s, " - "function ID: %s," - "function Name: %s," - "programming model: %s", - self.request_id, - function_id, - function_name, - programming_model, - ) + logger.info('Successfully processed FunctionLoadRequest, ' + 'request ID: %s, ' + 'function ID: %s,' + 'function Name: %s,' + 'programming model: %s', + self.request_id, + function_id, + function_name, + programming_model) return protos.StreamingMessage( request_id=self.request_id, function_load_response=protos.FunctionLoadResponse( function_id=function_id, - result=protos.StatusResult(status=protos.StatusResult.Success), - ), - ) + result=protos.StatusResult( + status=protos.StatusResult.Success))) except Exception as ex: return protos.StreamingMessage( @@ -628,10 +584,7 @@ async def _handle__function_load_request(self, request): function_id=function_id, result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex), - ), - ), - ) + exception=self._serialize_exception(ex)))) async def _handle__invocation_request(self, request): invocation_time = datetime.utcnow() @@ -647,30 +600,31 @@ async def _handle__invocation_request(self, request): current_task.set_azure_invocation_id(invocation_id) try: - fi: functions.FunctionInfo = self._functions.get_function(function_id) + fi: functions.FunctionInfo = self._functions.get_function( + function_id) assert fi is not None function_invocation_logs: List[str] = [ - "Received FunctionInvocationRequest", - f"request ID: {self.request_id}", - f"function ID: {function_id}", - f"function name: {fi.name}", - f"invocation ID: {invocation_id}", + 'Received FunctionInvocationRequest', + f'request ID: {self.request_id}', + f'function ID: {function_id}', + f'function name: {fi.name}', + f'invocation ID: {invocation_id}', f'function type: {"async" if fi.is_async else "sync"}', - f"timestamp (UTC): {invocation_time}", + f'timestamp (UTC): {invocation_time}' ] if not fi.is_async: function_invocation_logs.append( - f"sync threadpool max workers: " f"{self.get_sync_tp_workers_set()}" + f'sync threadpool max workers: ' + f'{self.get_sync_tp_workers_set()}' ) - logger.info(", ".join(function_invocation_logs)) + logger.info(', '.join(function_invocation_logs)) args = {} - http_v2_enabled = ( - self._functions.get_function(function_id).is_http_func - and HttpV2Registry.http_v2_enabled() - ) + http_v2_enabled = self._functions.get_function(function_id) \ + .is_http_func and \ + HttpV2Registry.http_v2_enabled() for pb in invoc_request.input_data: pb_type_info = fi.input_types[pb.name] @@ -685,21 +639,21 @@ async def _handle__invocation_request(self, request): trigger_metadata=trigger_metadata, pytype=pb_type_info.pytype, shmem_mgr=self._shmem_mgr, - function_name=self._functions.get_function(function_id).name, - is_deferred_binding=pb_type_info.deferred_bindings_enabled, - ) + function_name=self._functions.get_function( + function_id).name, + is_deferred_binding=pb_type_info.deferred_bindings_enabled) if http_v2_enabled: http_request = await http_coordinator.get_http_request_async( - invocation_id - ) + invocation_id) trigger_arg_name = fi.trigger_metadata.get('param_name') func_http_request = args[trigger_arg_name] await sync_http_request(http_request, func_http_request) args[trigger_arg_name] = http_request - fi_context = self._get_context(invoc_request, fi.name, fi.directory) + fi_context = self._get_context(invoc_request, fi.name, + fi.directory) # Use local thread storage to store the invocation ID # for a customer's threads @@ -715,22 +669,18 @@ async def _handle__invocation_request(self, request): if self._azure_monitor_available: self.configure_opentelemetry(fi_context) - call_result = await self._run_async_func(fi_context, fi.func, args) + call_result = \ + await self._run_async_func(fi_context, fi.func, args) else: call_result = await self._loop.run_in_executor( self._sync_call_tp, self._run_sync_func, - invocation_id, - fi_context, - fi.func, - args, - ) + invocation_id, fi_context, fi.func, args) if call_result is not None and not fi.has_return: raise RuntimeError( - f"function {fi.name!r} without a $return binding" - "returned a non-None value" - ) + f'function {fi.name!r} without a $return binding' + 'returned a non-None value') if http_v2_enabled: http_coordinator.set_http_response(invocation_id, call_result) @@ -746,13 +696,10 @@ async def _handle__invocation_request(self, request): continue param_binding = bindings.to_outgoing_param_binding( - out_type_info.binding_name, - val, + out_type_info.binding_name, val, pytype=out_type_info.pytype, - out_name=out_name, - shmem_mgr=self._shmem_mgr, - is_function_data_cache_enabled=cache_enabled, - ) + out_name=out_name, shmem_mgr=self._shmem_mgr, + is_function_data_cache_enabled=cache_enabled) output_data.append(param_binding) return_value = None @@ -771,10 +718,9 @@ async def _handle__invocation_request(self, request): invocation_response=protos.InvocationResponse( invocation_id=invocation_id, return_value=return_value, - result=protos.StatusResult(status=protos.StatusResult.Success), - output_data=output_data, - ), - ) + result=protos.StatusResult( + status=protos.StatusResult.Success), + output_data=output_data)) except Exception as ex: if http_v2_enabled: @@ -786,10 +732,7 @@ async def _handle__invocation_request(self, request): invocation_id=invocation_id, result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex), - ), - ), - ) + exception=self._serialize_exception(ex)))) async def _handle__function_environment_reload_request(self, request): """Only runs on Linux Consumption placeholder specialization. @@ -797,17 +740,16 @@ async def _handle__function_environment_reload_request(self, request): worker init request will be called directly. """ try: - logger.info( - "Received FunctionEnvironmentReloadRequest, " - "request ID: %s, " - "App Settings state: %s. " - "To enable debug level logging, please refer to " - "https://aka.ms/python-enable-debug-logging", - self.request_id, - get_python_appsetting_state(), - ) - - func_env_reload_request = request.function_environment_reload_request + logger.info('Received FunctionEnvironmentReloadRequest, ' + 'request ID: %s, ' + 'App Settings state: %s. ' + 'To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', + self.request_id, + get_python_appsetting_state()) + + func_env_reload_request = \ + request.function_environment_reload_request directory = func_env_reload_request.function_app_directory # Append function project root to module finding sys.path @@ -831,8 +773,8 @@ async def _handle__function_environment_reload_request(self, request): # Apply PYTHON_THREADPOOL_THREAD_COUNT self._stop_sync_call_tp() - self._sync_call_tp = self._create_sync_call_tp( - self._get_sync_tp_max_workers() + self._sync_call_tp = ( + self._create_sync_call_tp(self._get_sync_tp_max_workers()) ) if config_manager.is_envvar_true(PYTHON_ENABLE_DEBUG_LOGGING): @@ -847,20 +789,17 @@ async def _handle__function_environment_reload_request(self, request): bindings.load_binding_registry() capabilities = {} - if config_manager.get_app_setting( - setting=PYTHON_ENABLE_OPENTELEMETRY, - default_value=PYTHON_ENABLE_OPENTELEMETRY_DEFAULT, - ): + if config_manager.is_envvar_true(PYTHON_ENABLE_OPENTELEMETRY): self.initialize_azure_monitor() if self._azure_monitor_available: - capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = _TRUE + capabilities[constants.WORKER_OPEN_TELEMETRY_ENABLED] = ( + _TRUE) if config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( - directory, caller_info="environment_reload_request" - ) + directory, caller_info="environment_reload_request") if HttpV2Registry.http_v2_enabled(): capabilities[HTTP_URI] = \ @@ -872,69 +811,65 @@ async def _handle__function_environment_reload_request(self, request): self._function_metadata_exception = ex # Change function app directory - if getattr(func_env_reload_request, "function_app_directory", None): - self._change_cwd(func_env_reload_request.function_app_directory) + if getattr(func_env_reload_request, + 'function_app_directory', None): + self._change_cwd( + func_env_reload_request.function_app_directory) success_response = protos.FunctionEnvironmentReloadResponse( capabilities=capabilities, worker_metadata=self.get_worker_metadata(), - result=protos.StatusResult(status=protos.StatusResult.Success), - ) + result=protos.StatusResult(status=protos.StatusResult.Success)) return protos.StreamingMessage( request_id=self.request_id, - function_environment_reload_response=success_response, - ) + function_environment_reload_response=success_response) except Exception as ex: failure_response = protos.FunctionEnvironmentReloadResponse( result=protos.StatusResult( status=protos.StatusResult.Failure, - exception=self._serialize_exception(ex), - ) - ) + exception=self._serialize_exception(ex))) return protos.StreamingMessage( request_id=self.request_id, - function_environment_reload_response=failure_response, - ) + function_environment_reload_response=failure_response) def index_functions(self, function_path: str, function_dir: str): indexed_functions = loader.index_function_app(function_path) logger.info( - "Indexed function app and found %s functions", len(indexed_functions) + "Indexed function app and found %s functions", + len(indexed_functions) ) if indexed_functions: - fx_metadata_results, fx_bindings_logs = loader.process_indexed_function( - self._functions, indexed_functions, function_dir - ) + fx_metadata_results, fx_bindings_logs = ( + loader.process_indexed_function( + self._functions, + indexed_functions, + function_dir)) indexed_function_logs: List[str] = [] indexed_function_bindings_logs = [] for func in indexed_functions: func_binding_logs = fx_bindings_logs.get(func) for binding in func.get_bindings(): - deferred_binding_info = ( - func_binding_logs.get(binding.name) - if func_binding_logs.get(binding.name) - else "" - ) - indexed_function_bindings_logs.append( - (binding.type, binding.name, deferred_binding_info) - ) - - function_log = "Function Name: {}, Function Binding: {}".format( - func.get_function_name(), indexed_function_bindings_logs - ) + deferred_binding_info = func_binding_logs.get( + binding.name) \ + if func_binding_logs.get(binding.name) else "" + indexed_function_bindings_logs.append(( + binding.type, binding.name, deferred_binding_info)) + + function_log = "Function Name: {}, Function Binding: {}" \ + .format(func.get_function_name(), + indexed_function_bindings_logs) indexed_function_logs.append(function_log) logger.info( - "Successfully processed FunctionMetadataRequest for " - "functions: %s. Deferred bindings enabled: %s.", - " ".join(indexed_function_logs), - self._functions.deferred_bindings_enabled(), - ) + 'Successfully processed FunctionMetadataRequest for ' + 'functions: %s. Deferred bindings enabled: %s.', " ".join( + indexed_function_logs), + self._functions.deferred_bindings_enabled()) return fx_metadata_results @@ -960,73 +895,59 @@ async def _handle__close_shared_memory_resources_request(self, request): for map_name in map_names: try: to_delete_resources = not self._function_data_cache_enabled - success = self._shmem_mgr.free_mem_map( - map_name, to_delete_resources - ) + success = self._shmem_mgr.free_mem_map(map_name, + to_delete_resources) results[map_name] = success except Exception as e: - logger.error( - "Cannot free memory map %s - %s", map_name, e, exc_info=True - ) + logger.error('Cannot free memory map %s - %s', map_name, e, + exc_info=True) finally: response = protos.CloseSharedMemoryResourcesResponse( - close_map_results=results - ) + close_map_results=results) return protos.StreamingMessage( request_id=self.request_id, - close_shared_memory_resources_response=response, - ) + close_shared_memory_resources_response=response) def configure_opentelemetry(self, invocation_context): - carrier = { - _TRACEPARENT: invocation_context.trace_context.trace_parent, - _TRACESTATE: invocation_context.trace_context.trace_state, - } + carrier = {_TRACEPARENT: invocation_context.trace_context.trace_parent, + _TRACESTATE: invocation_context.trace_context.trace_state} ctx = self._trace_context_propagator.extract(carrier) self._context_api.attach(ctx) @staticmethod - def _get_context( - invoc_request: protos.InvocationRequest, name: str, directory: str - ) -> bindings.Context: - """For more information refer: + def _get_context(invoc_request: protos.InvocationRequest, name: str, + directory: str) -> bindings.Context: + """ For more information refer: https://aka.ms/azfunc-invocation-context """ trace_context = bindings.TraceContext( invoc_request.trace_context.trace_parent, invoc_request.trace_context.trace_state, - invoc_request.trace_context.attributes, - ) + invoc_request.trace_context.attributes) retry_context = bindings.RetryContext( invoc_request.retry_context.retry_count, invoc_request.retry_context.max_retry_count, - invoc_request.retry_context.exception, - ) + invoc_request.retry_context.exception) return bindings.Context( - name, - directory, - invoc_request.invocation_id, - _invocation_id_local, - trace_context, - retry_context, - ) + name, directory, invoc_request.invocation_id, + _invocation_id_local, trace_context, retry_context) @disable_feature_by(PYTHON_ROLLBACK_CWD_PATH) def _change_cwd(self, new_cwd: str): if os.path.exists(new_cwd): os.chdir(new_cwd) - logger.info("Changing current working directory to %s", new_cwd) + logger.info('Changing current working directory to %s', new_cwd) else: - logger.warning("Directory %s is not found when reloading", new_cwd) + logger.warning('Directory %s is not found when reloading', new_cwd) def _stop_sync_call_tp(self): """Deallocate the current synchronous thread pool and assign self._sync_call_tp to None. If the thread pool does not exist, this will be a no op. """ - if getattr(self, "_sync_call_tp", None): + if getattr(self, '_sync_call_tp', None): self._sync_call_tp.shutdown() self._sync_call_tp = None @@ -1036,49 +957,46 @@ def tp_max_workers_validator(value: str) -> bool: try: int_value = int(value) except ValueError: - logger.warning("%s must be an integer", PYTHON_THREADPOOL_THREAD_COUNT) + logger.warning('%s must be an integer', + PYTHON_THREADPOOL_THREAD_COUNT) return False if int_value < PYTHON_THREADPOOL_THREAD_COUNT_MIN: logger.warning( - "%s must be set to a value between %s and sys.maxint. " - "Reverting to default value for max_workers", + '%s must be set to a value between %s and sys.maxint. ' + 'Reverting to default value for max_workers', PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_THREADPOOL_THREAD_COUNT_MIN, - ) + PYTHON_THREADPOOL_THREAD_COUNT_MIN) return False return True # Starting Python 3.9, worker won't be putting a limit on the # max_workers count in the created threadpool. - default_value = ( - None - if sys.version_info.minor == 9 - else f"{PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT}" - ) + default_value = None if sys.version_info.minor == 9 \ + else f'{PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT}' max_workers = config_manager.get_app_setting( setting=PYTHON_THREADPOOL_THREAD_COUNT, default_value=default_value, - validator=tp_max_workers_validator, - ) - logger.info(f"max_workers: {max_workers}") + validator=tp_max_workers_validator) if sys.version_info.minor <= 7: - max_workers = min(int(max_workers), PYTHON_THREADPOOL_THREAD_COUNT_MAX_37) + max_workers = min(int(max_workers), + PYTHON_THREADPOOL_THREAD_COUNT_MAX_37) # We can box the app setting as int for earlier python versions. return int(max_workers) if max_workers else None def _create_sync_call_tp( - self, max_worker: Optional[int] - ) -> concurrent.futures.Executor: + self, max_worker: Optional[int]) -> concurrent.futures.Executor: """Create a thread pool executor with max_worker. This is a wrapper over ThreadPoolExecutor constructor. Consider calling this method after _stop_sync_call_tp() to ensure only 1 synchronous thread pool is running. """ - return concurrent.futures.ThreadPoolExecutor(max_workers=max_worker) + return concurrent.futures.ThreadPoolExecutor( + max_workers=max_worker + ) def _run_sync_func(self, invocation_id, context, func, params): # This helper exists because we need to access the current @@ -1087,7 +1005,8 @@ def _run_sync_func(self, invocation_id, context, func, params): try: if self._azure_monitor_available: self.configure_opentelemetry(context) - return ExtensionManager.get_sync_invocation_wrapper(context, func)(params) + return ExtensionManager.get_sync_invocation_wrapper(context, + func)(params) finally: context.thread_local_storage.invocation_id = None @@ -1099,20 +1018,24 @@ async def _run_async_func(self, context, func, params): def __poll_grpc(self): options = [] if self._grpc_max_msg_len: - options.append(("grpc.max_receive_message_length", self._grpc_max_msg_len)) - options.append(("grpc.max_send_message_length", self._grpc_max_msg_len)) + options.append(('grpc.max_receive_message_length', + self._grpc_max_msg_len)) + options.append(('grpc.max_send_message_length', + self._grpc_max_msg_len)) - channel = grpc.insecure_channel(f"{self._host}:{self._port}", options) + channel = grpc.insecure_channel( + f'{self._host}:{self._port}', options) try: grpc.channel_ready_future(channel).result( - timeout=self._grpc_connect_timeout - ) + timeout=self._grpc_connect_timeout) except Exception as ex: - self._loop.call_soon_threadsafe(self._grpc_connected_fut.set_exception, ex) + self._loop.call_soon_threadsafe( + self._grpc_connected_fut.set_exception, ex) return else: - self._loop.call_soon_threadsafe(self._grpc_connected_fut.set_result, True) + self._loop.call_soon_threadsafe( + self._grpc_connected_fut.set_result, True) stub = protos.FunctionRpcStub(channel) @@ -1128,17 +1051,14 @@ def gen(resp_queue): try: for req in grpc_req_stream: self._loop.call_soon_threadsafe( - self._loop.create_task, self._dispatch_grpc_request(req) - ) + self._loop.create_task, self._dispatch_grpc_request(req)) except Exception as ex: if ex is grpc_req_stream: # Yes, this is how grpc_req_stream iterator exits. return error_logger.exception( - "unhandled error in gRPC thread. Exception: {0}".format( - format_exception(ex) - ) - ) + 'unhandled error in gRPC thread. Exception: {0}'.format( + format_exception(ex))) raise @@ -1159,15 +1079,12 @@ def emit(self, record: LogRecord) -> None: # Logging such of an issue will cause infinite loop of gRPC logging # To mitigate, we should suppress the 2nd level error logging here # and use print function to report exception instead. - print( - f"{CONSOLE_LOG_PREFIX} ERROR: {str(runtime_error)}", - file=sys.stderr, - flush=True, - ) + print(f'{CONSOLE_LOG_PREFIX} ERROR: {str(runtime_error)}', + file=sys.stderr, flush=True) class ContextEnabledTask(asyncio.Task): - AZURE_INVOCATION_ID = "__azure_function_invocation_id__" + AZURE_INVOCATION_ID = '__azure_function_invocation_id__' def __init__(self, coro, loop, context=None): # The context param is only available for 3.11+. If @@ -1179,7 +1096,8 @@ def __init__(self, coro, loop, context=None): current_task = asyncio.current_task(loop) if current_task is not None: - invocation_id = getattr(current_task, self.AZURE_INVOCATION_ID, None) + invocation_id = getattr( + current_task, self.AZURE_INVOCATION_ID, None) if invocation_id is not None: self.set_azure_invocation_id(invocation_id) @@ -1192,13 +1110,13 @@ def get_current_invocation_id() -> Optional[str]: if loop is not None: current_task = asyncio.current_task(loop) if current_task is not None: - task_invocation_id = getattr( - current_task, ContextEnabledTask.AZURE_INVOCATION_ID, None - ) + task_invocation_id = getattr(current_task, + ContextEnabledTask.AZURE_INVOCATION_ID, + None) if task_invocation_id is not None: return task_invocation_id - return getattr(_invocation_id_local, "invocation_id", None) + return getattr(_invocation_id_local, 'invocation_id', None) _invocation_id_local = threading.local() diff --git a/azure_functions_worker/utils/app_setting_manager.py b/azure_functions_worker/utils/app_setting_manager.py index 9dcdb7dbc..c148357d9 100644 --- a/azure_functions_worker/utils/app_setting_manager.py +++ b/azure_functions_worker/utils/app_setting_manager.py @@ -20,17 +20,16 @@ def get_python_appsetting_state(): current_vars = config_manager.get_config() - python_specific_settings = [ - PYTHON_ROLLBACK_CWD_PATH, - PYTHON_THREADPOOL_THREAD_COUNT, - PYTHON_ISOLATE_WORKER_DEPENDENCIES, - PYTHON_ENABLE_DEBUG_LOGGING, - PYTHON_ENABLE_WORKER_EXTENSIONS, - FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, - PYTHON_SCRIPT_FILE_NAME, - PYTHON_ENABLE_INIT_INDEXING, - PYTHON_ENABLE_OPENTELEMETRY, - ] + python_specific_settings = \ + [PYTHON_ROLLBACK_CWD_PATH, + PYTHON_THREADPOOL_THREAD_COUNT, + PYTHON_ISOLATE_WORKER_DEPENDENCIES, + PYTHON_ENABLE_DEBUG_LOGGING, + PYTHON_ENABLE_WORKER_EXTENSIONS, + FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED, + PYTHON_SCRIPT_FILE_NAME, + PYTHON_ENABLE_INIT_INDEXING, + PYTHON_ENABLE_OPENTELEMETRY] app_setting_states = "".join( f"{app_setting}: {current_vars[app_setting]} | " @@ -39,16 +38,14 @@ def get_python_appsetting_state(): ) # Special case for extensions - if "PYTHON_ENABLE_WORKER_EXTENSIONS" not in current_vars: + if 'PYTHON_ENABLE_WORKER_EXTENSIONS' not in current_vars: if sys.version_info.minor == 9: - app_setting_states += ( - f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " - f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39)}" - ) + app_setting_states += \ + (f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " + f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39)}") else: - app_setting_states += ( - f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " - f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT)}" - ) + app_setting_states += \ + (f"{PYTHON_ENABLE_WORKER_EXTENSIONS}: " + f"{str(PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT)}") return app_setting_states diff --git a/azure_functions_worker/utils/config_manager.py b/azure_functions_worker/utils/config_manager.py index c16706cc2..3588f4de6 100644 --- a/azure_functions_worker/utils/config_manager.py +++ b/azure_functions_worker/utils/config_manager.py @@ -34,24 +34,10 @@ def read_config(self, function_path: str): except FileNotFoundError: pass - # updates the config dictionary with the environment variables - # this prioritizes set env variables over the config file - # env_copy = os.environ - # for k, v in env_copy.items(): - # self.config_data.update({k.upper(): v}) - def set_config(self, function_path: str): self.read_config(function_path) self.read_environment_variables() - def config_exists(self) -> bool: - if self.config_data is None: - self.set_config("") - return self.config_data is not None - - def get_config(self) -> dict: - return self.config_data - def is_true_like(self, setting: str) -> bool: if setting is None: return False @@ -66,11 +52,6 @@ def is_false_like(self, setting: str) -> bool: def is_envvar_true(self, key: str) -> bool: key_upper = key.upper() - # special case for PYTHON_ENABLE_DEBUG_LOGGING - # This is read by the host and must be set in os.environ - if key_upper == "PYTHON_ENABLE_DEBUG_LOGGING": - val = os.getenv(key_upper) - return self.is_true_like(val) if self.config_exists() and not self.config_data.get(key_upper): return False return self.is_true_like(self.config_data.get(key_upper)) @@ -137,5 +118,13 @@ def clear_config(self): self.config_data.clear() self.config_data = None + def config_exists(self) -> bool: + if self.config_data is None: + self.set_config("") + return self.config_data is not None + + def get_config(self) -> dict: + return self.config_data + config_manager = ConfigManager() diff --git a/tests/unittests/test_app_setting_manager.py b/tests/unittests/test_app_setting_manager.py index 4d437001c..e4428d35a 100644 --- a/tests/unittests/test_app_setting_manager.py +++ b/tests/unittests/test_app_setting_manager.py @@ -72,7 +72,6 @@ def setUpClass(cls): cls._patch_environ = patch.dict('os.environ', os_environ) cls._patch_environ.start() super().setUpClass() - config_manager.clear_config() config_manager.read_environment_variables() @classmethod diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 5a68bc4db..f179893e4 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -176,7 +176,6 @@ async def test_dispatcher_sync_threadpool_set_worker(self): await self._check_if_function_is_ok(host) await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_invalid_worker_count(self): """Test when sync threadpool maximum worker is set to an invalid value, @@ -197,7 +196,6 @@ async def test_dispatcher_sync_threadpool_invalid_worker_count(self): self._default_workers) mock_logger.warning.assert_any_call( '%s must be an integer', PYTHON_THREADPOOL_THREAD_COUNT) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_below_min_setting(self): """Test if the sync threadpool will pick up default value when the @@ -216,7 +214,6 @@ async def test_dispatcher_sync_threadpool_below_min_setting(self): 'Reverting to default value for max_workers', PYTHON_THREADPOOL_THREAD_COUNT, PYTHON_THREADPOOL_THREAD_COUNT_MIN) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_exceed_max_setting(self): """Test if the sync threadpool will pick up default max value when the @@ -233,7 +230,6 @@ async def test_dispatcher_sync_threadpool_exceed_max_setting(self): # Ensure the dispatcher sync threadpool should fallback to max await self._assert_workers_threadpool(self._ctrl, host, self._allowed_max_workers) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_dispatcher_sync_threadpool_in_placeholder(self): """Test if the sync threadpool will pick up app setting in placeholder @@ -381,7 +377,6 @@ async def test_sync_invocation_request_log_threads(self): r'\d{2}:\d{2}:\d{2}.\d{6}), ' 'sync threadpool max workers: 5' ) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_async_invocation_request_log_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -406,7 +401,6 @@ async def test_async_invocation_request_log_threads(self): r'(\d{4}-\d{2}-\d{2} ' r'\d{2}:\d{2}:\d{2}.\d{6})' ) - os.environ.pop(PYTHON_THREADPOOL_THREAD_COUNT) async def test_sync_invocation_request_log_in_placeholder_threads(self): with patch('azure_functions_worker.dispatcher.logger') as mock_logger: @@ -604,7 +598,6 @@ class TestDispatcherStein(testutils.AsyncTestCase): def setUp(self): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_STEIN_FUNCTIONS_DIR) - config_manager.clear_config() async def test_dispatcher_functions_metadata_request(self): """Test if the functions metadata response will be sent correctly @@ -736,7 +729,6 @@ async def test_dispatcher_load_modules_dedicated_app(self): "working_directory: , Linux Consumption: False," " Placeholder: False", logs ) - os.environ.pop("PYTHON_ISOLATE_WORKER_DEPENDENCIES") async def test_dispatcher_load_modules_con_placeholder_enabled(self): """Test modules are loaded in consumption apps with placeholder mode diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 8bf016557..9a0eb2c6d 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -80,7 +80,6 @@ def setUp(self): def tearDown(self) -> None: os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) - self.mock_sys_path.stop() self.mock_sys_module.stop() self.mock_environ.stop() diff --git a/tests/unittests/test_third_party_http_functions.py b/tests/unittests/test_third_party_http_functions.py index ad8408655..ba1e44f2c 100644 --- a/tests/unittests/test_third_party_http_functions.py +++ b/tests/unittests/test_third_party_http_functions.py @@ -5,7 +5,6 @@ import pathlib import re import typing - from unittest.mock import patch from tests.utils import testutils diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index a1a23677b..d2f5c3634 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -83,7 +83,6 @@ def setUp(self): self.mock_environ.start() self.mock_sys_module.start() self.mock_sys_path.start() - config_manager.clear_config() def tearDown(self): self.mock_sys_path.stop() From 1ce6f5122a59ea670b1208247bc2783f0cb82def Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Wed, 18 Sep 2024 10:03:34 -0500 Subject: [PATCH 32/33] formatting fixes --- azure_functions_worker/dispatcher.py | 55 +++++++++++--------- azure_functions_worker/logging.py | 1 + tests/unittests/test_dispatcher.py | 1 + tests/unittests/test_extension.py | 1 + tests/unittests/test_script_file_name.py | 1 + tests/unittests/test_utilities_dependency.py | 3 +- 6 files changed, 35 insertions(+), 27 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 753e6e543..dd4b36d37 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -140,6 +140,7 @@ def get_sync_tp_workers_set(self): """We don't know the exact value of the threadcount set for the Python 3.9 scenarios (as we'll start passing only None by default), and we need to get that information. + Ref: concurrent.futures.thread.ThreadPoolExecutor.__init__._max_workers """ return self._sync_call_tp._max_workers @@ -193,19 +194,19 @@ async def dispatch_forever(self): # sourcery skip: swap-if-expression root_logger.setLevel(log_level) root_logger.addHandler(logging_handler) - logger.info("Switched to gRPC logging.") + logger.info('Switched to gRPC logging.') logging_handler.flush() try: await forever finally: - logger.warning("Detaching gRPC logging due to exception.") + logger.warning('Detaching gRPC logging due to exception.') logging_handler.flush() root_logger.removeHandler(logging_handler) # Reenable console logging when there's an exception enable_console_logging() - logger.warning("Switched to console logging due to exception.") + logger.warning('Switched to console logging due to exception.') finally: DispatcherMeta.__current_dispatcher__ = None @@ -235,12 +236,12 @@ def on_logging(self, record: logging.LogRecord, elif record.levelno >= logging.DEBUG: log_level = protos.RpcLog.Debug else: - log_level = getattr(protos.RpcLog, "None") + log_level = getattr(protos.RpcLog, 'None') if is_system_log_category(record.name): - log_category = protos.RpcLog.RpcLogCategory.Value("System") + log_category = protos.RpcLog.RpcLogCategory.Value('System') else: # customers using logging will yield 'root' in record.name - log_category = protos.RpcLog.RpcLogCategory.Value("User") + log_category = protos.RpcLog.RpcLogCategory.Value('User') log = dict( level=log_level, @@ -251,7 +252,7 @@ def on_logging(self, record: logging.LogRecord, invocation_id = get_current_invocation_id() if invocation_id is not None: - log["invocation_id"] = invocation_id + log['invocation_id'] = invocation_id self._grpc_resp_queue.put_nowait( protos.StreamingMessage( @@ -270,7 +271,7 @@ def worker_id(self) -> str: @staticmethod def _serialize_exception(exc: Exception): try: - message = f"{type(exc).__name__}: {exc}" + message = f'{type(exc).__name__}: {exc}' except Exception: message = ('Unhandled exception in function. ' 'Could not serialize original exception message.') @@ -278,13 +279,13 @@ def _serialize_exception(exc: Exception): try: stack_trace = marshall_exception_trace(exc) except Exception: - stack_trace = "" + stack_trace = '' return protos.RpcException(message=message, stack_trace=stack_trace) async def _dispatch_grpc_request(self, request): - content_type = request.WhichOneof("content") - request_handler = getattr(self, f"_handle__{content_type}", None) + content_type = request.WhichOneof('content') + request_handler = getattr(self, f'_handle__{content_type}', None) if request_handler is None: # Don't crash on unknown messages. Some of them can be ignored; # and if something goes really wrong the host can always just @@ -356,16 +357,16 @@ def update_opentelemetry_status(self): async def _handle__worker_init_request(self, request): worker_init_request = request.worker_init_request config_manager.set_config( - os.path.join(worker_init_request.function_app_directory, "az-config.json") + os.path.join(worker_init_request.function_app_directory, 'az-config.json') ) logger.info( - "Received WorkerInitRequest, " - "python version %s, " - "worker version %s, " - "request ID %s. " - "App Settings state: %s. " - "To enable debug level logging, please refer to " - "https://aka.ms/python-enable-debug-logging", + 'Received WorkerInitRequest, ' + 'python version %s, ' + 'worker version %s, ' + 'request ID %s. ' + 'App Settings state: %s. ' + 'To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', sys.version, VERSION, self.request_id, @@ -441,7 +442,7 @@ def load_function_metadata(self, function_app_directory, caller_info): """ script_file_name = config_manager.get_app_setting( setting=PYTHON_SCRIPT_FILE_NAME, - default_value=f"{PYTHON_SCRIPT_FILE_NAME_DEFAULT}") + default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}') logger.debug( 'Received load metadata request from %s, request ID %s, ' @@ -469,7 +470,7 @@ async def _handle__functions_metadata_request(self, request): script_file_name) logger.info( - "Received WorkerMetadataRequest, request ID %s, " "function_path: %s", + 'Received WorkerMetadataRequest, request ID %s, ' 'function_path: %s', self.request_id, function_path, ) @@ -659,7 +660,7 @@ async def _handle__invocation_request(self, request): # for a customer's threads fi_context.thread_local_storage.invocation_id = invocation_id if fi.requires_context: - args["context"] = fi_context + args['context'] = fi_context if fi.output_types: for name in fi.output_types: @@ -767,7 +768,7 @@ async def _handle__function_environment_reload_request(self, request): os.environ[var] = env_vars[var] config_manager.set_config( os.path.join( - func_env_reload_request.function_app_directory, "az-config.json" + func_env_reload_request.function_app_directory, 'az-config.json' ) ) @@ -799,7 +800,8 @@ async def _handle__function_environment_reload_request(self, request): if config_manager.is_envvar_true(PYTHON_ENABLE_INIT_INDEXING): try: self.load_function_metadata( - directory, caller_info="environment_reload_request") + directory, + caller_info="environment_reload_request") if HttpV2Registry.http_v2_enabled(): capabilities[HTTP_URI] = \ @@ -819,7 +821,8 @@ async def _handle__function_environment_reload_request(self, request): success_response = protos.FunctionEnvironmentReloadResponse( capabilities=capabilities, worker_metadata=self.get_worker_metadata(), - result=protos.StatusResult(status=protos.StatusResult.Success)) + result=protos.StatusResult( + status=protos.StatusResult.Success)) return protos.StreamingMessage( request_id=self.request_id, @@ -855,7 +858,7 @@ def index_functions(self, function_path: str, function_dir: str): func_binding_logs = fx_bindings_logs.get(func) for binding in func.get_bindings(): deferred_binding_info = func_binding_logs.get( - binding.name) \ + binding.name)\ if func_binding_logs.get(binding.name) else "" indexed_function_bindings_logs.append(( binding.type, binding.name, deferred_binding_info)) diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py index d86c39c81..adb5ff294 100644 --- a/azure_functions_worker/logging.py +++ b/azure_functions_worker/logging.py @@ -13,6 +13,7 @@ SDK_LOG_PREFIX = "azure.functions" SYSTEM_ERROR_LOG_PREFIX = "azure_functions_worker_errors" + logger: logging.Logger = logging.getLogger(SYSTEM_LOG_PREFIX) error_logger: logging.Logger = ( logging.getLogger(SYSTEM_ERROR_LOG_PREFIX)) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index f179893e4..0afc5212d 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -598,6 +598,7 @@ class TestDispatcherStein(testutils.AsyncTestCase): def setUp(self): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_STEIN_FUNCTIONS_DIR) + config_manager.clear_config() async def test_dispatcher_functions_metadata_request(self): """Test if the functions metadata response will be sent correctly diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 9a0eb2c6d..8bf016557 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -80,6 +80,7 @@ def setUp(self): def tearDown(self) -> None: os.environ.pop(PYTHON_ENABLE_WORKER_EXTENSIONS) + self.mock_sys_path.stop() self.mock_sys_module.stop() self.mock_environ.stop() diff --git a/tests/unittests/test_script_file_name.py b/tests/unittests/test_script_file_name.py index 6d969ebeb..0437cf429 100644 --- a/tests/unittests/test_script_file_name.py +++ b/tests/unittests/test_script_file_name.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os + from tests.utils import testutils from azure_functions_worker.constants import ( diff --git a/tests/unittests/test_utilities_dependency.py b/tests/unittests/test_utilities_dependency.py index fe4ac57f2..8e9f3f576 100644 --- a/tests/unittests/test_utilities_dependency.py +++ b/tests/unittests/test_utilities_dependency.py @@ -6,9 +6,10 @@ import unittest from unittest.mock import patch +from tests.utils import testutils + from azure_functions_worker.utils.dependency import DependencyManager from azure_functions_worker.utils.config_manager import config_manager -from tests.utils import testutils class TestDependencyManager(unittest.TestCase): From 193282cadeed1ebd947bd9f4b44ed2811d5bd19d Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Thu, 19 Sep 2024 14:32:47 -0500 Subject: [PATCH 33/33] formatting fixes --- azure_functions_worker/dispatcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index dd4b36d37..9372d573c 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -470,7 +470,8 @@ async def _handle__functions_metadata_request(self, request): script_file_name) logger.info( - 'Received WorkerMetadataRequest, request ID %s, ' 'function_path: %s', + 'Received WorkerMetadataRequest, request ID %s, ' + 'function_path: %s', self.request_id, function_path, )