From 6a1db185310bfe8d4c3c7d4bcbf5116386c549a2 Mon Sep 17 00:00:00 2001 From: Angelina Kharchevnikova Date: Tue, 16 Oct 2018 18:06:26 +0300 Subject: [PATCH] Fastboot check (#44) * Add fastboot size check * Add fastboot size check after build stage * Add fastboot size check after build stage * Add fastboot size check after build stage Fix dir * Add fastboot size check after build stage Fix dir * Add fastboot size check after build stage Fix dir Fix function call * Add fastboot size check after build stage Fix dir Fix function call Fix max lib size * Add fastboot size check after build stage Get lib version dynamically * Fixes after code review * Fixes for api version * Add format_map to check_lib_size function. Updated documentation. * Add format_map to check_lib_size function * Move library size check to Install stage * Add set_env in Install as workaround * Add set_env in Install as workaround * Add set_env in Install as workaround * Fix for lib size * Increase max lib size * Change max_size according current fastboot lib size --- conf_how_to.py | 8 +++ conf_linux_public.py | 138 ++++++++++++++++++++++++++++--------------- 2 files changed, 100 insertions(+), 46 deletions(-) diff --git a/conf_how_to.py b/conf_how_to.py index 193396f..9ccac46 100644 --- a/conf_how_to.py +++ b/conf_how_to.py @@ -146,6 +146,14 @@ def action(name, stage='build', cmd=None, work_dir=None, env=None, script=None, """ action('script calling', cmd=f'bash -c "my_script.sh"') +""" +If you need access to variable that is initialized during action execution, you should use str.format_map() +""" +action('build', cmd=('strip ./__bin/release/libmfxhw64-fastboot.so.{ENV[API_VERSION]}')) +""" +Here we add options[ENV][API_VERSION] to 'lib_path' +""" +lib_path = str(lib_path).format_map(options) # ============================================================================== # Configuration: archiving diff --git a/conf_linux_public.py b/conf_linux_public.py index ad533a9..57f7b2d 100644 --- a/conf_linux_public.py +++ b/conf_linux_public.py @@ -18,51 +18,71 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +PRODUCT_REPOS = [ + {'name': 'MediaSDK'}, + # Give possibility to build linux for changes from product configs repository + # This repo not needed for build and added only to support CI process + {'name': 'product-configs'} + #{'name': 'flow_test'}, +] -#TODO: move functions to the shared module -def set_env(repo_path, gcc_latest, clang_version): - def _get_commit_number(repo_path): - if not repo_path.exists(): - return '0' - import git - git_repo = git.Git(str(repo_path)) - return str(git_repo.rev_list('--count', 'HEAD')) - - def _get_api_version(repo_path): - """ - :param name: Path to the MediaSDK folder - :type name: String or Path - - Function finds the lines like: - `#define MFX_VERSION_MAJOR 1` - `#define MFX_VERSION_MINOR 26` - And prints the version like: - `1.26` - """ - import re - import pathlib - - mediasdk_api_header = pathlib.Path(repo_path) / 'api' / 'include' / 'mfxdefs.h' - if not mediasdk_api_header.exists(): - return '0' - - with open(mediasdk_api_header, 'r') as lines: - major_version = "" - minor_version = "" - for line in lines: - major_version_pattern = re.search("MFX_VERSION_MAJOR\s(\d+)", line) - if major_version_pattern: - major_version = major_version_pattern.group(1) - continue - - minor_version_pattern = re.search("MFX_VERSION_MINOR\s(\d+)", line) - if minor_version_pattern: - minor_version = minor_version_pattern.group(1) - - if major_version and minor_version: - return f"{major_version}.{minor_version}" - raise Exception(f"API_VERSION did not found in {mediasdk_api_header}") +ENABLE_DEVTOOLSET = 'source /opt/rh/devtoolset-6/enable' +GCC_LATEST = '8.2.0' +options["STRIP_BINARIES"] = True +MEDIA_SDK_REPO_DIR = options.get('REPOS_DIR') / PRODUCT_REPOS[0]['name'] +MEDIA_SDK_BUILD_DIR = options.get('BUILD_DIR') +# Max size = current fastboot lib size + ~50Kb +FASTBOOT_LIB_MAX_SIZE = 1 * 1024 * 1024 + 256 * 1024 # byte + + +def get_commit_number(repo_path=MEDIA_SDK_REPO_DIR): + if not repo_path.exists(): + return '0' + import git + git_repo = git.Git(str(repo_path)) + return str(git_repo.rev_list('--count', 'HEAD')) + + +def get_api_version(repo_path=MEDIA_SDK_REPO_DIR): + """ + :param name: Path to the MediaSDK folder + :type name: String or Path + + Function finds the lines like: + `#define MFX_VERSION_MAJOR 1` + `#define MFX_VERSION_MINOR 26` + And prints the version like: + `1.26` + """ + import re + import pathlib + + mediasdk_api_header = pathlib.Path(repo_path) / 'api' / 'include' / 'mfxdefs.h' + if not mediasdk_api_header.exists(): + raise Exception(f"No {mediasdk_api_header.name} found in {mediasdk_api_header.parent}") + + with open(mediasdk_api_header, 'r') as lines: + major_version = "" + minor_version = "" + for line in lines: + major_version_pattern = re.search("MFX_VERSION_MAJOR\s(\d+)", line) + if major_version_pattern: + major_version = major_version_pattern.group(1) + continue + + minor_version_pattern = re.search("MFX_VERSION_MINOR\s(\d+)", line) + if minor_version_pattern: + minor_version = minor_version_pattern.group(1) + + if major_version and minor_version: + return f"{major_version}.{minor_version}" + raise Exception(f"API_VERSION did not found in {mediasdk_api_header}") + + +# TODO: move functions to the shared module +def set_env(repo_path, gcc_latest, clang_version, _get_commit_number=get_commit_number, _get_api_version=get_api_version): + api_ver = _get_api_version(repo_path) build_num = _get_commit_number(repo_path) @@ -74,18 +94,18 @@ def _get_api_version(repo_path): options["ENV"]['MFX_VP8_VERSION'] = f'{plugin_version}' options["ENV"]['MFX_VP9_VERSION'] = f'{plugin_version}' options["ENV"]['MFX_H264LA_VERSION'] = f'{plugin_version}' - options["ENV"]['MFX_HOME'] = f'{str(repo_path)}' compiler_version = args.get('compiler_version') if args.get('compiler') == "gcc" and compiler_version == gcc_latest: options["ENV"]['CC'] = '/usr/bin/gcc-8' options["ENV"]['CXX'] = '/usr/bin/g++-8' - + elif args.get('compiler') == "clang" and compiler_version == clang_version: options["ENV"]['CC'] = f'/usr/bin/clang-{compiler_version}' options["ENV"]['CXX'] = f'/usr/bin/clang++-{compiler_version}' + #TODO: add more smart logic or warnings?! (potential danger zone) def get_building_cmd(command, gcc_latest, enable_devtoolset): # Ubuntu Server: gcc_latest or clang @@ -94,6 +114,21 @@ def get_building_cmd(command, gcc_latest, enable_devtoolset): else: return f'{enable_devtoolset} && {command}' #enable new compiler on CentOS +def check_lib_size(threshold_size, lib_path): + """ + :param lib_path: path to lib + :return: pathlib.Path + """ + + import pathlib + + lib_path = pathlib.Path(str(lib_path).format_map(options)) + current_lib_size = lib_path.stat().st_size + log.info(f'{lib_path} size = {current_lib_size}byte\n') + if current_lib_size > threshold_size: + if not options['STRIP_BINARIES']: + log.warning("Library size could exceed threshold because stripping build binaries option is OFF") + raise Exception(f"{lib_path.name} size = {current_lib_size}byte exceeds max_size = {threshold_size}byte") # Choose repository in accordance with prefix of product type if product_type.startswith("public"): @@ -149,8 +184,9 @@ def get_building_cmd(command, gcc_latest, enable_devtoolset): #Additional (custom) options (they extend default parameters): if args.get('fastboot'): - fastboot_cmake_path = MEDIA_SDK_REPO_DIR / 'builder' / 'profiles' / 'fastboot.cmake' + fastboot_cmake_path = MEDIA_SDK_REPO_DIR / 'builder/profiles/fastboot.cmake' cmake_command.append(f'-DMFX_CONFIG_FILE={fastboot_cmake_path}') + if args.get('api_latest'): cmake_command.append('-DAPI:STRING=latest') @@ -183,6 +219,16 @@ def get_building_cmd(command, gcc_latest, enable_devtoolset): stage=stage.INSTALL, cmd=get_building_cmd(f'make DESTDIR={options["INSTALL_DIR"]} install', GCC_LATEST, ENABLE_DEVTOOLSET)) +if args.get('fastboot'): + # TODO: Pass data between stages with pickle in build scripts instead + action('count api version and build number', + stage=stage.INSTALL, + callfunc=(set_env, [MEDIA_SDK_REPO_DIR, GCC_LATEST, CLANG_VERSION], {})) + + action('check fastboot lib size', + stage=stage.INSTALL, + callfunc=(check_lib_size, [FASTBOOT_LIB_MAX_SIZE, MEDIA_SDK_BUILD_DIR / '__bin/release/libmfxhw64-fastboot.so.{ENV[API_VERSION]}'], {})) + DEV_PKG_DATA_TO_ARCHIVE.extend([ {