inputs: allow to override repo ref with sha1 form store

.github/workflows/build_all.yml

@@ -30,6 +30,23 @@ on:
         type: boolean
         required: false
         default: false
+
+      use_sha1_from_live:
+        type: boolean
+        required: false
+        default: false
+      provider:
+        type: string
+        required: false
+        default: "1"
+      device:
+        type: string
+        required: false
+        default: "nanos+"
+      version:
+        type: string
+        required: false
+        default: "1.1.0"
 
 jobs:
   setup-devices:
@@ -168,8 +185,14 @@ jobs:
           name: input_${{ matrix.index }}.json
 
       - name: Setup repos
+        if: ${{ inputs.use_sha1_from_live }}
+        run: | 
+          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} 
+
+      - name: Setup repos
+        if: ${{ !inputs.use_sha1_from_live }}
         run: | 
-          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json 
+          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json
 
       - name: Launch build
         run: | 

scripts/build_and_test/build_app.py

@@ -1,9 +1,15 @@
 from pathlib import Path
 from device import Devices, Device
 from utils import run_cmd
+import os
 
 def build_variant(target: str, sdk_path: str, variant_param: str, variant_value: str, app_build_path:
         Path, extra_flags: str=""):
+
+    if not os.path.exists(app_build_path):
+        print("\t=> KO")
+        return True, f"Error: {app_build_path} does not exists\n"
+
     error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True)
     if variant_param:
         error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", cwd=app_build_path, no_throw=True)

scripts/build_and_test/main.py

@@ -6,6 +6,7 @@
 from scan_app import scan_all_devices
 from device import Devices
 from utils import git_setup, merge_json
+from sha1 import override_sha1
 
 SDK_NAME = "sdk"
 SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git"
@@ -37,6 +38,11 @@
             default=Path("output_files/error_logs.txt"))
     parser.add_argument("--workdir", required=False, type=str, default="workdir")
 
+    parser.add_argument("--use_sha1_from_live", required=False, action='store_true')
+    parser.add_argument("--provider", required=False, type=str)
+    parser.add_argument("--device", required=False, type=str)
+    parser.add_argument("--version", required=False, type=str)
+
     args = parser.parse_args()
 
     abs_workdir = Path.cwd()/args.workdir
@@ -71,6 +77,20 @@
         print("Error: input file does not exist")
         exit()
 
+    if args.use_sha1_from_live:
+        if not args.provider:
+            print("Error: you must specify provider")
+            exit()
+        if not args.device:
+            print("Error: you must specify device")
+            exit()
+        if not args.version:
+            print("Error: you must specify version")
+            exit()
+
+        input_json = override_sha1(input_json, args.provider, args.device, args.version)
+
+
     git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir)
 
     output = {}

scripts/build_and_test/sha1.py

@@ -0,0 +1,36 @@
+import requests
+import json
+
+STORE_API = "https://appstore.aws.prd.ldg-tech.com"
+
+
+def fetch_data_from_api():
+    request_headers = {'Content-Type': 'application/json'}
+    r = requests.get(STORE_API + '/api/applications', headers=request_headers)
+
+    store_app_list = json.loads(r.text)
+    return store_app_list
+
+
+def get_sha1(live_json: dict, provider: str, device: str, version: str):
+    for version_json in live_json["application_versions"]:
+        pattern = f"{device}/{version}/"
+
+        if version_json["firmware"].startswith(pattern):
+            if int(provider) in version_json["providers"]:
+                return version_json["sha1"]
+            return
+
+
+def override_sha1(input_json: dict, provider: str, device: str, version: str):
+    live_json = fetch_data_from_api()
+
+    for app_json in input_json:
+        for live_app_json in live_json:
+            if app_json["url"] == live_app_json["sourceURL"]:
+                sha1 = get_sha1(live_app_json, provider, device, version)
+                if sha1:
+                    app_json["ref"] = sha1
+                break
+
+    return input_json

scripts/build_and_test/utils.py

@@ -1,4 +1,5 @@
 import subprocess
+import shutil
 from pathlib import Path
 
 
@@ -37,12 +38,22 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path):
     GIT_CONFIG = ' -c  url."https://github.com/".insteadOf="[email protected]:" -c url."https://".insteadOf="git://"'
 
     if not Path.exists(workdir/Path(repo_name)):
-        run_cmd(f"git {GIT_CONFIG} clone {repo_url} --recurse-submodules {repo_name}", cwd=workdir)
+        run_cmd(f"git {GIT_CONFIG} clone {repo_url} {repo_name}", cwd=workdir)
     else:
         run_cmd(f"git fetch", cwd=workdir/Path(repo_name))
 
-    run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name))
-    run_cmd("git submodule update --recursive", cwd=workdir/Path(repo_name))
+    error, _ = run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name), no_throw=True)
+    if error:
+        print("Error: removing folder")
+        shutil.rmtree(workdir/Path(repo_name))
+        return
+
+    error, _ = run_cmd(f"git {GIT_CONFIG} submodule update --init --recursive", cwd=workdir/Path(repo_name), no_throw=True)
+    if error:
+        print("Error: removing folder")
+        shutil.rmtree(workdir/Path(repo_name))
+        return
+
 
 def merge_json(json1: dict, json2: dict, key: str):
     merged_data = []