diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index 8260499..3025817 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -100,8 +100,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -186,17 +186,17 @@ jobs: - name: Setup repos if: ${{ inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} - name: Setup repos if: ${{ !inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch build - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -534,8 +534,8 @@ jobs: name: build_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" - name: Archive output file uses: actions/upload-artifact@v3 @@ -577,12 +577,12 @@ jobs: name: merged_build_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_build_output.json --output_file out.md --key build + python3 scripts/entrypoint.py convert_output --input_file full_build_output.json --output_file out.md --key build cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -590,7 +590,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -603,7 +603,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_build_output.json --key build + python3 scripts/entrypoint.py status_output --input_file full_build_output.json --key build - name: Archive output file if: always() diff --git a/.github/workflows/fast-checks.yml b/.github/workflows/fast-checks.yml index 00c24a7..99183cd 100644 --- a/.github/workflows/fast-checks.yml +++ b/.github/workflows/fast-checks.yml @@ -12,12 +12,6 @@ jobs: lint: strategy: fail-fast: false - matrix: - directory: - - build_and_test - - create_app_list - - output_scripts - - split_input name: Linting runs-on: ubuntu-latest steps: @@ -25,17 +19,11 @@ jobs: uses: actions/checkout@v3 - run: pip install flake8 - name: Flake8 lint Python code - run: find scripts/${{ matrix.directory }} -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' + run: find scripts/ -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' mypy: strategy: fail-fast: false - matrix: - directory: - - build_and_test - - create_app_list - - output_scripts - - split_input name: Type checking runs-on: ubuntu-latest steps: @@ -43,7 +31,7 @@ jobs: uses: actions/checkout@v3 - run: pip install mypy types-requests - name: Mypy type checking - run: mypy scripts/${{ matrix.directory }} + run: mypy scripts/ misspell: strategy: diff --git a/.github/workflows/refresh_inputs.yml b/.github/workflows/refresh_inputs.yml index bc18a5d..703a580 100644 --- a/.github/workflows/refresh_inputs.yml +++ b/.github/workflows/refresh_inputs.yml @@ -20,14 +20,14 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest - + steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Build db - run: | - python3 scripts/create_app_list/main.py --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} + run: | + python3 scripts/entrypoint.py create_app_list --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} - name: Upload file uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/scan_all.yml b/.github/workflows/scan_all.yml index 5802697..a5ca7f2 100644 --- a/.github/workflows/scan_all.yml +++ b/.github/workflows/scan_all.yml @@ -83,8 +83,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -168,12 +168,12 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch scan - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -254,8 +254,8 @@ jobs: name: scan_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" - name: Archive output file uses: actions/upload-artifact@v3 @@ -554,12 +554,12 @@ jobs: name: merged_scan_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_scan_output.json --output_file out.md --key scan + python3 scripts/entrypoint.py convert_output --input_file full_scan_output.json --output_file out.md --key scan cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -567,7 +567,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -580,7 +580,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_scan_output.json --key scan + python3 scripts/entrypoint.py status_output --input_file full_scan_output.json --key scan - name: Archive output file uses: actions/upload-artifact@v3 diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml index 538333b..7e3b642 100644 --- a/.github/workflows/test_all.yml +++ b/.github/workflows/test_all.yml @@ -47,8 +47,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -136,8 +136,8 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Install tests dependencies run: | @@ -167,22 +167,22 @@ jobs: - name: Launch test nanosp if: ${{ env.test_nanosp }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt - name: Launch test nanox if: ${{ env.test_nanox }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt - name: Launch test stax if: ${{ env.test_stax }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 with: @@ -339,8 +339,8 @@ jobs: name: test_output_10.json - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_output_*.json" --output_file test_output.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_output_*.json" --output_file test_output.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 @@ -351,7 +351,7 @@ jobs: - name: Convert to markdown run: | - python scripts/output_scripts/convert.py --input_file test_output.json --output_file out.md --key test + python scripts/entrypoint.py convert_output --input_file test_output.json --output_file out.md --key test cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -359,7 +359,7 @@ jobs: - name: Convert to slack json run: | - python scripts/output_scripts/slack.py --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} + python scripts/entrypoint.py slack_output --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -372,4 +372,4 @@ jobs: - name: Set job status run: | - python scripts/output_scripts/status.py --input_file test_output.json --key test + python scripts/entrypoint.py status_output --input_file test_output.json --key test diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py index e69de29..eb36c52 100644 --- a/scripts/build_and_test/__init__.py +++ b/scripts/build_and_test/__init__.py @@ -0,0 +1,104 @@ +import json +from pathlib import Path +from argparse import Namespace + +from build_and_test.sha1 import override_sha1 +from build_and_test.build_app import build_all_devices +from build_and_test.test_app import test_all_devices +from build_and_test.scan_app import scan_all_devices +from build_and_test.device import Devices +from build_and_test.utils import git_setup, merge_json + +SDK_NAME = "sdk" +SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" +SDK_BRANCH = "origin/master" + + +def main(args: Namespace) -> None: + input_json = {} + + abs_workdir = Path.cwd() / args.workdir + + if not abs_workdir.exists(): + abs_workdir.mkdir() + + nanos_enable = False + nanosp_enable = False + nanox_enable = False + stax_enable = False + + if args.all or args.nanos: + print("Nanos enabled") + nanos_enable = True + if args.all or args.nanosp: + print("Nanosp enabled") + nanosp_enable = True + if args.all or args.nanox: + print("Nanox enabled") + nanox_enable = True + if args.all or args.stax: + print("Stax enabled") + stax_enable = True + + devices = Devices(nanos_enable, nanosp_enable, nanox_enable, stax_enable) + + if Path(args.input_file).exists(): + with open(args.input_file) as json_file: + input_json = json.load(json_file) + else: + print("Error: input file does not exist") + exit() + + if args.use_sha1_from_live: + if not args.provider: + print("Error: you must specify provider") + exit() + if not args.device: + print("Error: you must specify device") + exit() + if not args.version: + print("Error: you must specify version") + exit() + + input_json = override_sha1(input_json, args.provider, args.device, args.version) + + git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) + + output = {} + test_output = {} + build_output = [] + logs = "" + + for app_json in input_json: + repo_name = app_json.get("name") + if not args.skip_setup: + repo_ref = app_json.get("ref") + repo_url = app_json.get("url") + print(f"Setup {repo_name}") + git_setup(repo_name, repo_ref, repo_url, abs_workdir) + + if args.build: + print(f"Build {repo_name}") + build_app, log = build_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(build_app) + logs += log + + if args.test: + print(f"Test {repo_name}") + test_app, log = test_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(test_app) + logs += log + + if args.scan_build: + print(f"Scan build {repo_name}") + scan_app, log = scan_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(scan_app) + logs += log + + output = merge_json(build_output, test_output, "name") + + with open(args.output_file, 'w') as json_file: + json.dump(output, json_file, indent=1) + + with open(args.logs_file, 'w') as file: + file.write(logs) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index ec62293..6dc9f6e 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -1,7 +1,8 @@ -from pathlib import Path -from device import Devices, Device -from utils import run_cmd import os +from pathlib import Path + +from build_and_test.device import Devices, Device +from build_and_test.utils import run_cmd def build_variant(target: str, diff --git a/scripts/build_and_test/main.py b/scripts/build_and_test/main.py deleted file mode 100644 index f1818ae..0000000 --- a/scripts/build_and_test/main.py +++ /dev/null @@ -1,133 +0,0 @@ -import json -from pathlib import Path -from argparse import ArgumentParser -from sha1 import override_sha1 - -from build_app import build_all_devices -from test_app import test_all_devices -from scan_app import scan_all_devices -from device import Devices -from utils import git_setup, merge_json - -SDK_NAME = "sdk" -SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" -SDK_BRANCH = "origin/master" - - -if __name__ == "__main__": - parser = ArgumentParser() - input_json = {} - - parser.add_argument("--skip_setup", action='store_true') - - # Devices - parser.add_argument("--all", action='store_true') - parser.add_argument("--nanos", action='store_true') - parser.add_argument("--nanosp", action='store_true') - parser.add_argument("--nanox", action='store_true') - parser.add_argument("--stax", action='store_true') - - parser.add_argument("--test", action='store_true') - parser.add_argument("--build", action='store_true') - parser.add_argument("--scan_build", action='store_true') - - parser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") - - parser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) - parser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) - parser.add_argument("--logs_file", required=False, type=Path, - default=Path("output_files/error_logs.txt")) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - - parser.add_argument("--use_sha1_from_live", required=False, action='store_true') - parser.add_argument("--provider", required=False, type=str) - parser.add_argument("--device", required=False, type=str) - parser.add_argument("--version", required=False, type=str) - - args = parser.parse_args() - - abs_workdir = Path.cwd()/args.workdir - - if not abs_workdir.exists(): - abs_workdir.mkdir() - - nanos_enable = False - nanosp_enable = False - nanox_enable = False - stax_enable = False - - if args.all or args.nanos: - print("Nanos enabled") - nanos_enable = True - if args.all or args.nanosp: - print("Nanosp enabled") - nanosp_enable = True - if args.all or args.nanox: - print("Nanox enabled") - nanox_enable = True - if args.all or args.stax: - print("Stax enabled") - stax_enable = True - - devices = Devices(nanos_enable, nanosp_enable, nanox_enable, stax_enable) - - if Path(args.input_file).exists(): - with open(args.input_file) as json_file: - input_json = json.load(json_file) - else: - print("Error: input file does not exist") - exit() - - if args.use_sha1_from_live: - if not args.provider: - print("Error: you must specify provider") - exit() - if not args.device: - print("Error: you must specify device") - exit() - if not args.version: - print("Error: you must specify version") - exit() - - input_json = override_sha1(input_json, args.provider, args.device, args.version) - - git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) - - output = {} - test_output = {} - build_output = [] - logs = "" - - for app_json in input_json: - repo_name = app_json.get("name") - if not args.skip_setup: - repo_ref = app_json.get("ref") - repo_url = app_json.get("url") - print(f"Setup {repo_name}") - git_setup(repo_name, repo_ref, repo_url, abs_workdir) - - if args.build: - print(f"Build {repo_name}") - build_app, log = build_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(build_app) - logs += log - - if args.test: - print(f"Test {repo_name}") - test_app, log = test_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(test_app) - logs += log - - if args.scan_build: - print(f"Scan build {repo_name}") - scan_app, log = scan_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(scan_app) - logs += log - - output = merge_json(build_output, test_output, "name") - - with open(args.output_file, 'w') as json_file: - json.dump(output, json_file, indent=1) - - with open(args.logs_file, 'w') as file: - file.write(logs) diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 07df451..23615a9 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -1,6 +1,7 @@ from pathlib import Path -from device import Devices, Device -from utils import run_cmd + +from build_and_test.device import Devices, Device +from build_and_test.utils import run_cmd def scan_variant(target: str, diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index c149a8d..34a9d78 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -1,9 +1,10 @@ from pathlib import Path -from device import Devices, Device -from build_app import build_variant -from utils import run_cmd from typing import Tuple +from build_and_test.device import Devices, Device +from build_and_test.build_app import build_variant +from build_and_test.utils import run_cmd + def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str) -> Tuple[str, str]: output: str diff --git a/scripts/create_app_list/main.py b/scripts/create_app_list/__init__.py old mode 100755 new mode 100644 similarity index 61% rename from scripts/create_app_list/main.py rename to scripts/create_app_list/__init__.py index 8e7bcd9..51e610f --- a/scripts/create_app_list/main.py +++ b/scripts/create_app_list/__init__.py @@ -1,27 +1,13 @@ -from argparse import ArgumentParser +from argparse import Namespace from pathlib import Path import json -from parse_github import parse_github +from create_app_list.parse_github import parse_github +from create_app_list.gen_variant import gen_variant +from create_app_list.utils import git_setup, merge_json -from gen_variant import gen_variant -from utils import git_setup, merge_json - -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--access_token", required=True, type=str) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - parser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) - parser.add_argument("--repo_file", required=False, type=Path) - parser.add_argument("--skip_setup", required=False, action='store_true') - - parser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) - parser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) - parser.add_argument("--variant_output_file", required=False, type=Path, default=Path("output_files/variant.json")) - - args = parser.parse_args() +def main(args: Namespace) -> None: abs_workdir = Path.cwd()/args.workdir if not abs_workdir.exists(): diff --git a/scripts/create_app_list/app_load_params_utils.py b/scripts/create_app_list/app_load_params_utils.py index a8d603f..5853479 100755 --- a/scripts/create_app_list/app_load_params_utils.py +++ b/scripts/create_app_list/app_load_params_utils.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import json from pathlib import Path from typing import Dict diff --git a/scripts/create_app_list/gen_variant.py b/scripts/create_app_list/gen_variant.py index 9262e62..1c5f6a9 100644 --- a/scripts/create_app_list/gen_variant.py +++ b/scripts/create_app_list/gen_variant.py @@ -1,10 +1,8 @@ -#!/usr/bin/env python3 - from pathlib import Path from collections import namedtuple from typing import Dict, List, Union -from makefile_dump import get_app_listvariants +from create_app_list.makefile_dump import get_app_listvariants Models = namedtuple('Models', ['sdk_value', 'device_name']) diff --git a/scripts/create_app_list/makefile_dump.py b/scripts/create_app_list/makefile_dump.py index cf0bdf9..b6fa6e2 100755 --- a/scripts/create_app_list/makefile_dump.py +++ b/scripts/create_app_list/makefile_dump.py @@ -1,15 +1,14 @@ -#!/usr/bin/env python3 - """ This is a pure duplicate of https://github.com/LedgerHQ/ledger-app-workflows/blob/master/scripts/makefile_dump.py This is to allow easily generating the db from the apps code. """ -from utils import run_cmd from pathlib import Path from typing import Tuple, List +from create_app_list.utils import run_cmd + def get_app_listvariants(app_build_path: Path, sdk: str = "$NANOS_SDK", diff --git a/scripts/entrypoint.py b/scripts/entrypoint.py new file mode 100644 index 0000000..dc78881 --- /dev/null +++ b/scripts/entrypoint.py @@ -0,0 +1,106 @@ +import sys +from argparse import ArgumentParser, Namespace +from pathlib import Path + +sys.path.insert(1, Path(__file__).resolve().parent) + + +def parse_args() -> Namespace: + parser = ArgumentParser() + subparsers = parser.add_subparsers(help="Specific operation", dest="operation") + + # split_input + subparser = subparsers.add_parser('split_input') + subparser.add_argument("--split_count", required=False, type=int, default=10) + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/input.json")) + + # build_and_test + subparser = subparsers.add_parser('build_and_test') + subparser.add_argument("--skip_setup", action='store_true') + + subparser.add_argument("--all", action='store_true') + subparser.add_argument("--nanos", action='store_true') + subparser.add_argument("--nanosp", action='store_true') + subparser.add_argument("--nanox", action='store_true') + subparser.add_argument("--stax", action='store_true') + + subparser.add_argument("--test", action='store_true') + subparser.add_argument("--build", action='store_true') + subparser.add_argument("--scan_build", action='store_true') + + subparser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") + + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) + subparser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) + subparser.add_argument("--logs_file", required=False, type=Path, + default=Path("output_files/error_logs.txt")) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + + subparser.add_argument("--use_sha1_from_live", required=False, action='store_true') + subparser.add_argument("--provider", required=False, type=str) + subparser.add_argument("--device", required=False, type=str) + subparser.add_argument("--version", required=False, type=str) + + # output_scripts + # # convert + subparser = subparsers.add_parser('convert_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + # # merge + subparser = subparsers.add_parser('merge_output', description="Merge JSON files based on a specified key") + subparser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") + subparser.add_argument("--output_file", help="Output merged JSON file") + subparser.add_argument("--key", help="Key to use for merging") + # # status + subparser = subparsers.add_parser('status_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--key", required=True) + # # slack + subparser = subparsers.add_parser('slack_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + subparser.add_argument("--devices", required=False, type=str) + subparser.add_argument("--url", required=False, type=str) + + # create_app_list + subparser = subparsers.add_parser('create_app_list') + subparser.add_argument("--access_token", required=True, type=str) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + subparser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) + subparser.add_argument("--repo_file", required=False, type=Path) + subparser.add_argument("--skip_setup", required=False, action='store_true') + + subparser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) + subparser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) + subparser.add_argument("--variant_output_file", required=False, type=Path, + default=Path("output_files/variant.json")) + + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + if args.operation == 'split_input': + import split_input + split_input.main(args) + elif args.operation == 'build_and_test': + import build_and_test + build_and_test.main(args) + elif args.operation == 'convert_output': + import output_scripts.convert + output_scripts.convert.main(args) + elif args.operation == 'merge_output': + import output_scripts.merge + output_scripts.merge.main(args) + elif args.operation == 'status_output': + import output_scripts.status + output_scripts.status.main(args) + elif args.operation == 'slack_output': + import output_scripts.slack + output_scripts.slack.main(args) + elif args.operation == 'create_app_list': + import create_app_list + create_app_list.main(args) diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py old mode 100755 new mode 100644 index 55766f8..118358a --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def json_to_markdown(json_list, key): @@ -69,15 +68,7 @@ def count_status(json_list, key): """ -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) diff --git a/scripts/output_scripts/merge.py b/scripts/output_scripts/merge.py index 346a62b..083721e 100644 --- a/scripts/output_scripts/merge.py +++ b/scripts/output_scripts/merge.py @@ -1,6 +1,6 @@ -import argparse import glob import json +from argparse import Namespace def merge_jsons(json1, json2, key): @@ -38,14 +38,7 @@ def merge_multiple_jsons(input_files, key): return result -def main(): - parser = argparse.ArgumentParser(description="Merge JSON files based on a specified key") - parser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") - parser.add_argument("--output_file", help="Output merged JSON file") - parser.add_argument("--key", help="Key to use for merging") - - args = parser.parse_args() - +def main(args: Namespace) -> None: input_files = glob.glob(args.input_pattern) if not input_files: print("No input files found.") @@ -54,7 +47,3 @@ def main(): merged_json = merge_multiple_jsons(input_files, args.key) with open(args.output_file, 'w') as f: json.dump(merged_json, f, indent=1) - - -if __name__ == "__main__": - main() diff --git a/scripts/output_scripts/slack.py b/scripts/output_scripts/slack.py index 91142bc..7e309a1 100644 --- a/scripts/output_scripts/slack.py +++ b/scripts/output_scripts/slack.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def count_test_status(json_list): @@ -66,17 +65,7 @@ def count_status(json_list, key): return success_count, fail_count, total_count, fail_list -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - parser.add_argument("--devices", required=False, type=str) - parser.add_argument("--url", required=False, type=str) - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) diff --git a/scripts/output_scripts/status.py b/scripts/output_scripts/status.py old mode 100755 new mode 100644 index bc3defd..3c15f20 --- a/scripts/output_scripts/status.py +++ b/scripts/output_scripts/status.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def check_status(json_data, key): @@ -19,15 +18,7 @@ def check_status(json_data, key): raise ValueError(f"Failed for {app_name}") -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--key", required=True) - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_data = json.load(json_file) - check_status(json_data, args.key) diff --git a/scripts/split_input/split_input.py b/scripts/split_input.py old mode 100755 new mode 100644 similarity index 70% rename from scripts/split_input/split_input.py rename to scripts/split_input.py index 035d69a..4559cc3 --- a/scripts/split_input/split_input.py +++ b/scripts/split_input.py @@ -1,18 +1,11 @@ import json from pathlib import Path -from argparse import ArgumentParser +from argparse import Namespace -if __name__ == "__main__": - parser = ArgumentParser() +def main(args: Namespace) -> None: input_json = {} - parser.add_argument("--input_file", required=False, type=Path, - default=Path("input_files/input.json")) - parser.add_argument("--split_count", required=False, type=Path, default=10) - - args = parser.parse_args() - if Path(args.input_file).exists(): with open(args.input_file) as json_file: input_json = json.load(json_file) diff --git a/scripts/split_input/__init__.py b/scripts/split_input/__init__.py deleted file mode 100644 index e69de29..0000000