From 079b392e1036a4142f648347bc1e5d94766f8c15 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 7 Oct 2022 11:53:31 +0100 Subject: [PATCH] Cleanup of scripts --- cm-mlops/script/get-bazel/customize.py | 11 +++--- cm-mlops/script/get-bazel/run.sh | 3 +- cm-mlops/script/get-cmake/customize.py | 10 +++-- cm-mlops/script/get-cmake/run.sh | 3 +- cm-mlops/script/get-cuda/customize.py | 8 ++-- cm-mlops/script/get-cuda/run.sh | 3 +- cm-mlops/script/get-openssl/_cm.json | 12 ++++++ cm-mlops/script/get-openssl/customize.py | 37 +++++++++---------- cm-mlops/script/get-openssl/run.sh | 4 +- cm-mlops/script/install-bazel/customize.py | 3 +- .../install-cmake-prebuilt/customize.py | 3 ++ .../script/install-cuda-prebuilt/customize.py | 3 +- cm-mlops/script/install-openssl/customize.py | 9 ++--- 13 files changed, 62 insertions(+), 47 deletions(-) diff --git a/cm-mlops/script/get-bazel/customize.py b/cm-mlops/script/get-bazel/customize.py index a976faf7ae..18a33cfae0 100644 --- a/cm-mlops/script/get-bazel/customize.py +++ b/cm-mlops/script/get-bazel/customize.py @@ -11,9 +11,7 @@ def preprocess(i): file_name = 'bazel.exe' if os_info['platform'] == 'windows' else 'bazel' env['FILE_NAME'] = file_name - if 'CM_BAZEL_INSTALLED_PATH' in env: - found_path = env['CM_BAZEL_INSTALLED_PATH'] - else: + if 'CM_BAZEL_BIN_WITH_PATH' not in env: r = i['automation'].find_artifact({'file_name': file_name, 'env': env, 'os_info':os_info, @@ -29,8 +27,6 @@ def preprocess(i): else: return r - env['CM_LLVM_INSTALLED_PATH'] = r['found_path'] - return {'return':0} def detect_version(i): @@ -53,6 +49,11 @@ def postprocess(i): if r['return'] >0: return r version = r['version'] + found_file_path = env['CM_BAZEL_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + env['CM_BAZEL_INSTALLED_PATH'] = found_path + env['CM_BAZEL_CACHE_TAGS'] = 'version-'+version diff --git a/cm-mlops/script/get-bazel/run.sh b/cm-mlops/script/get-bazel/run.sh index 50679b9481..e145f46385 100644 --- a/cm-mlops/script/get-bazel/run.sh +++ b/cm-mlops/script/get-bazel/run.sh @@ -1,6 +1,5 @@ #!/bin/bash -bazel_bin=${CM_BAZEL_INSTALLED_PATH}/${FILE_NAME} -bazel_bin=${CM_BAZEL_BIN_WITH_PATH:-$bazel_bin} +bazel_bin=${CM_BAZEL_BIN_WITH_PATH} if [[ ${CM_VERSION} == "0.26.1" ]]; then ${bazel_bin} version |grep "Build label" |sed 's/Build label:/bazel/' > tmp-ver.out else diff --git a/cm-mlops/script/get-cmake/customize.py b/cm-mlops/script/get-cmake/customize.py index 97b5358f4d..e2f6d34b17 100644 --- a/cm-mlops/script/get-cmake/customize.py +++ b/cm-mlops/script/get-cmake/customize.py @@ -10,8 +10,7 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] file_name = 'cmake.exe' if os_info['platform'] == 'windows' else 'cmake' - env['FILE_NAME'] = file_name - if 'CM_CMAKE_INSTALLED_PATH' not in env: + if 'CM_CMAKE_BIN_WITH_PATH' not in env: r = i['automation'].find_artifact({'file_name': file_name, 'env':env, 'os_info':os_info, @@ -27,8 +26,6 @@ def preprocess(i): else: return r - env['CM_CMAKE_INSTALLED_PATH'] = r['found_path'] - return {'return':0} def detect_version(i): @@ -52,6 +49,11 @@ def postprocess(i): if r['return'] >0: return r version = r['version'] + found_file_path = env['CM_CMAKE_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + + env['CMAKE_INSTALLED_PATH'] = found_path env['CM_CMAKE_CACHE_TAGS'] = 'version-'+version if 'CM_HOST_CPU_TOTAL_CORES' in env: diff --git a/cm-mlops/script/get-cmake/run.sh b/cm-mlops/script/get-cmake/run.sh index 2432135b0e..6d2aeff974 100644 --- a/cm-mlops/script/get-cmake/run.sh +++ b/cm-mlops/script/get-cmake/run.sh @@ -1,6 +1,5 @@ #!/bin/bash -cmake_bin=${CM_CMAKE_INSTALLED_PATH}/${FILE_NAME} -cmake_bin=${CM_CMAKE_BIN_WITH_PATH:-${cmake_bin}} +cmake_bin=${CM_CMAKE_BIN_WITH_PATH} ${cmake_bin} --version > tmp-ver.out test $? -eq 0 || exit 1 diff --git a/cm-mlops/script/get-cuda/customize.py b/cm-mlops/script/get-cuda/customize.py index 171a1309d2..b8a50fbf30 100644 --- a/cm-mlops/script/get-cuda/customize.py +++ b/cm-mlops/script/get-cuda/customize.py @@ -10,7 +10,6 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] file_name = 'nvcc.exe' if os_info['platform'] == 'windows' else 'nvcc' - env['FILE_NAME'] = file_name if 'CM_CUDA_INSTALLED_PATH' not in env: r = i['automation'].find_artifact({'file_name': file_name, @@ -31,8 +30,6 @@ def preprocess(i): else: return r - env['CM_CUDA_INSTALLED_PATH'] = r['found_path'] - return {'return':0} def detect_version(i): @@ -53,6 +50,11 @@ def postprocess(i): env = i['env'] r = detect_version(i) if r['return'] >0: return r + found_file_path = env['CM_NVCC_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + env['CM_CUDA_INSTALLED_PATH'] = found_path + version = r['version'] diff --git a/cm-mlops/script/get-cuda/run.sh b/cm-mlops/script/get-cuda/run.sh index 20cbb47ee1..42077f038a 100644 --- a/cm-mlops/script/get-cuda/run.sh +++ b/cm-mlops/script/get-cuda/run.sh @@ -1,6 +1,5 @@ #!/bin/bash -nvcc_bin=${CM_CUDA_INSTALLED_PATH}/${FILE_NAME} -nvcc_bin=${CM_NVCC_BIN_WITH_PATH:-${nvcc_bin}} +nvcc_bin=${CM_NVCC_BIN_WITH_PATH} ${nvcc_bin} -V > tmp-ver.out test $? -eq 0 || exit 1 diff --git a/cm-mlops/script/get-openssl/_cm.json b/cm-mlops/script/get-openssl/_cm.json index fff9cb9bb4..6a70d11f45 100644 --- a/cm-mlops/script/get-openssl/_cm.json +++ b/cm-mlops/script/get-openssl/_cm.json @@ -14,5 +14,17 @@ "openssl", "lib-openssl" ], + "env": { + "CM_TMP_REQUIRE_INSTALL": "no" + }, + "prehook_deps": [ + { + "tags": "install,openssl", + "reuse_version": true, + "enable_if_env": { + "CM_TMP_REQUIRE_INSTALL": [ "yes" ] + } + } + ], "uid": "febdae70e9e64e30" } diff --git a/cm-mlops/script/get-openssl/customize.py b/cm-mlops/script/get-openssl/customize.py index 7a79ed14fa..b2bc29b3ca 100644 --- a/cm-mlops/script/get-openssl/customize.py +++ b/cm-mlops/script/get-openssl/customize.py @@ -9,37 +9,25 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] - file_name = 'openssl.exe' if os_info['platform'] == 'windows' else 'openssl' + file_name = 'openssl' r = i['automation'].find_artifact({'file_name': file_name, 'env': env, 'os_info':os_info, 'default_path_env_key': 'PATH', 'detect_version':True, - 'env_path_key':'CM_PYTHON_BIN_WITH_PATH', + 'env_path_key':'CM_OPENSSL_BIN_WITH_PATH', 'run_script_input':i['run_script_input'], 'recursion_spaces':i['recursion_spaces']}) if r['return']>0: - if r['return'] == 16 and os_info['platform'] != 'windows': - if env.get('CM_TMP_FAIL_IF_NOT_FOUND','').lower() == 'yes': - return r - - print (recursion_spaces+' # {}'.format(r['error'])) - - # Attempt to run installer - r = {'return':0, 'skip':True, 'script':{'tags':'install,openssl'}} - - return r - - found_path = r['found_path'] + if r['return'] == 16 and os_info['platform'] != 'windows': + env['CM_TMP_REQUIRE_INSTALL'] = "yes" + return {'return': 0} + return r return {'return':0} - -def postprocess(i): - - env = i['env'] - +def detect_version(i): r = i['automation'].parse_version({'match_text': r'OpenSSL\s*([\d.]+)', 'group_number': 1, 'env_key':'CM_OPENSSL_VERSION', @@ -49,8 +37,17 @@ def postprocess(i): version = r['version'] print (i['recursion_spaces'] + ' Detected version: {}'.format(version)) + return {'return':0, 'version':version} + + +def postprocess(i): + + env = i['env'] + r = detect_version(i) + if r['return'] >0: return r + version = r['version'] # Save tags that can be used to specialize further dependencies (such as python packages) tags = 'version-'+version - return {'return':0, 'version':version} + return {'return':0} diff --git a/cm-mlops/script/get-openssl/run.sh b/cm-mlops/script/get-openssl/run.sh index 15d2982ded..14277c91a0 100644 --- a/cm-mlops/script/get-openssl/run.sh +++ b/cm-mlops/script/get-openssl/run.sh @@ -1,4 +1,4 @@ #!/bin/bash - -openssl version > tmp-ver.out +openssl_bin=${CM_OPENSSL_BIN_WITH_PATH} +${openssl_bin} version > tmp-ver.out 2>/dev/null test $? -eq 0 || exit 1 diff --git a/cm-mlops/script/install-bazel/customize.py b/cm-mlops/script/install-bazel/customize.py index a54c6257e4..4d01d8a0e4 100644 --- a/cm-mlops/script/install-bazel/customize.py +++ b/cm-mlops/script/install-bazel/customize.py @@ -22,8 +22,9 @@ def preprocess(i): if 'CM_GIT_CHECKOUT' not in env: env['CM_GIT_CHECKOUT'] = 'releases/gcc-' + need_version - + bazel_bin = "bazel" env['CM_BAZEL_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') + env['CM_BAZEL_BIN_WITH_PATH'] = os.path.join(os.getcwd(), 'install', 'bin', bazel_bin) env['CM_TMP_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() return {'return':0} diff --git a/cm-mlops/script/install-cmake-prebuilt/customize.py b/cm-mlops/script/install-cmake-prebuilt/customize.py index a95ec9abe3..611184c0c5 100644 --- a/cm-mlops/script/install-cmake-prebuilt/customize.py +++ b/cm-mlops/script/install-cmake-prebuilt/customize.py @@ -93,5 +93,8 @@ def preprocess(i): env['CM_CMAKE_INSTALLED_PATH'] = path_bin env['CM_TMP_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() + bin_name = 'cmake.exe' if os_info['platform'] == 'windows' else 'cmake' + env['CM_CMAKE_BIN_WITH_PATH'] = os.path.join(path_bin, bin_name) + return {'return':0} diff --git a/cm-mlops/script/install-cuda-prebuilt/customize.py b/cm-mlops/script/install-cuda-prebuilt/customize.py index 942a88f29a..bfebfe5b7e 100644 --- a/cm-mlops/script/install-cuda-prebuilt/customize.py +++ b/cm-mlops/script/install-cuda-prebuilt/customize.py @@ -10,8 +10,9 @@ def preprocess(i): automation = i['automation'] recursion_spaces = i['recursion_spaces'] - + nvcc_bin = "nvcc" env['CM_CUDA_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install') + env['CM_NVCC_BIN_WITH_PATH'] = os.path.join(os.getcwd(), 'install', nvcc_bin) env['CM_TMP_GET_DEPENDENT_CACHED_PATH'] = os.getcwd() return {'return':0} diff --git a/cm-mlops/script/install-openssl/customize.py b/cm-mlops/script/install-openssl/customize.py index 39d37d7b82..ab76de1582 100644 --- a/cm-mlops/script/install-openssl/customize.py +++ b/cm-mlops/script/install-openssl/customize.py @@ -20,11 +20,6 @@ def preprocess(i): print (recursion_spaces + ' # Requested version: {}'.format(need_version)) - path_bin = os.path.join(os.getcwd(), 'install', 'bin') - - env['CM_TMP_PATH'] = path_bin - env['CM_TMP_FAIL_IF_NOT_FOUND'] = 'yes' - return {'return':0} def postprocess(i): @@ -37,4 +32,8 @@ def postprocess(i): if '+LD_LIBRARY_PATH' not in env: env['+LD_LIBRARY_PATH'] = [] env['+LD_LIBRARY_PATH'].append(path_lib) + bin_name = "openssl" + path_bin = os.path.join(os.getcwd(), 'install', 'bin') + env['CM_OPENSSL_INSTALLED_PATH'] = path_bin + env['CM_OPENSSL_BIN_WITH_PATH'] = os.path.join(path_bin, bin_name) return {'return':0}