From 681d17acfcf21a3934d90a93e606cdae4aab0a1d Mon Sep 17 00:00:00 2001 From: forefy Date: Tue, 13 Feb 2024 18:56:57 +0200 Subject: [PATCH] Flexible output location, Auto-installer improvements --- .github/workflows/pytest.yml | 38 ++++++++++++++++ eburger/main.py | 27 +++++++++-- eburger/utils/filesystem.py | 22 ++++++++- eburger/utils/helpers.py | 65 ++++++++++++++++++++++++++- eburger/utils/installers.py | 86 ++++++++++++++++++++++++++++++------ eburger/yaml_parser.py | 5 ++- tests/test_arguments.py | 5 ++- 7 files changed, 225 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..6de4f0d --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,38 @@ +name: pytest + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: [ cron: "0 7 * * 2" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m venv /tmp/pytest-env + source /tmp/pytest-env/bin/activate + python -m pip install --upgrade pip setuptools wheel + python -m pip install pytest + python -m pip install . + + - name: Run pytest + run: | + source /tmp/pytest-env/bin/activate + pytest -s tests/ diff --git a/eburger/main.py b/eburger/main.py index 820086e..0c22f23 100644 --- a/eburger/main.py +++ b/eburger/main.py @@ -12,6 +12,7 @@ get_foundry_ast_json, get_hardhat_ast_json, get_solidity_version_from_file, + roughly_check_valid_file_path_name, select_project, ) from eburger.utils.helpers import ( @@ -99,9 +100,11 @@ def main(): path_type = "hardhat" else: log( - "error", - f"{args.solidity_file_or_folder} is neither a file nor a directory.", + "info", + f"{args.solidity_file_or_folder} is neither a file nor a directory, please select a valid path.", ) + sys.exit(0) + elif args.ast_json_file: filename = args.ast_json_file filename = filename.replace(".json", "") # Clean possible file extension @@ -243,14 +246,30 @@ def main(): insights_json_path = settings.outputs_dir / f"eburger_output_{filename}.json" save_as_json(insights_json_path, analysis_output) - results_path = settings.project_root / "eburger-output.json" - if args.output == "sarif": + if roughly_check_valid_file_path_name(args.output): + custom_output_path = Path(args.output) + file_extension = custom_output_path.suffix[1:] + if file_extension not in ["json", "sarif", "md"]: + log( + "error", + f"Unrecognized output file extension provided. ({file_extension})", + ) + + results_path = Path.cwd() / custom_output_path + if file_extension == "sarif": + save_as_sarif(results_path, insights) + elif file_extension == "md": + save_as_markdown(results_path, analysis_output) + elif file_extension == "json": + save_as_json(results_path, analysis_output) + elif args.output == "sarif": results_path = settings.project_root / f"eburger-output.sarif" save_as_sarif(results_path, insights) elif args.output in ["markdown", "md"]: results_path = settings.project_root / f"eburger-output.md" save_as_markdown(results_path, analysis_output) else: + results_path = settings.project_root / "eburger-output.json" save_as_json(results_path, analysis_output) log( diff --git a/eburger/utils/filesystem.py b/eburger/utils/filesystem.py index 3d296e8..ce7ce3b 100644 --- a/eburger/utils/filesystem.py +++ b/eburger/utils/filesystem.py @@ -187,7 +187,12 @@ def find_recursive_files_by_patterns(source_path, patterns: list) -> list: continue filtered_paths.append(path) file_paths.extend(filtered_paths) - return list(set(file_paths)) + + unique_file_paths = list(set(file_paths)) + sorted_unique_file_paths = sorted( + unique_file_paths, key=lambda path: (len(str(path)), str(path)) + ) + return sorted_unique_file_paths def select_project(project_paths: list) -> Path: @@ -209,3 +214,18 @@ def select_project(project_paths: list) -> Path: else: print("Only one project available.") return Path(project_paths[0]) + + +def roughly_check_valid_file_path_name(path_str: str) -> bool: + if not path_str or path_str.isspace() or "." not in path_str: + return False + + if os.name == "nt": + invalid_chars = r'<>:"|?*\n\r\t' + else: + invalid_chars = "\0" + + if any(char in path_str for char in invalid_chars): + return False + + return True diff --git a/eburger/utils/helpers.py b/eburger/utils/helpers.py index ae4f72c..64cb3ca 100644 --- a/eburger/utils/helpers.py +++ b/eburger/utils/helpers.py @@ -1,5 +1,6 @@ from datetime import datetime import json +import os from pathlib import Path import re import shlex @@ -22,11 +23,16 @@ def is_valid_json(json_string: str) -> bool: def run_command( - command: str, directory: Path = None, shell: bool = False, live_output: bool = False -): + command: str, + directory: Path = None, + shell: bool = False, + live_output: bool = False, +) -> tuple[list, list]: log("info", f"{command}") + results = [] errors = [] + process = subprocess.Popen( command if shell else shlex.split(command), stdout=subprocess.PIPE, @@ -84,3 +90,58 @@ def get_filename_from_path(file_path: str) -> tuple: output_filename = settings.outputs_dir / f"{filename}.json" return filename, output_filename + + +# Emulated "source" command to allow subprocesses to reload terminal state without forcing the user to reload the terminal +def python_shell_source(execute_source: bool = True) -> tuple[str, str]: + shell = os.environ.get("SHELL", "") + home = os.environ.get("HOME", "") + source_command = "" + + if "zsh" in shell: + zdotdir = os.environ.get("ZDOTDIR", home) + profile = os.path.join(zdotdir, ".zshenv") + source_command = f"{profile}" + source_syntax = "source" + and_sign = "&&" + elif "bash" in shell: + profile = os.path.join(home, ".bashrc") + source_command = f"{profile}" + source_syntax = "source" + and_sign = "&&" + elif "fish" in shell: + profile = os.path.join(home, ".config/fish/config.fish") + source_command = f"{profile}" + source_syntax = "source" + and_sign = "; and" + elif "ash" in shell: + profile = os.path.join(home, ".profile") + source_command = f"{profile}" + source_syntax = "." + and_sign = "&&" + else: + log( + "error", + "Couldn't automatically install, please reload the current shell or install manually, and try again.", + ) + + if execute_source: + # e.g. 'source .zshenv && printenv' + constructed_source_command = ( + f"{source_syntax} {source_command} {and_sign} printenv" + ) + log("info", f"/bin/bash -c {constructed_source_command}") + pipe = subprocess.Popen( + ["/bin/bash", "-c", f"{constructed_source_command}"], + stdout=subprocess.PIPE, + text=True, + ) + env_lines = pipe.stdout.readlines() + env_dict = { + line.split("=", 1)[0]: line.split("=", 1)[1].strip() + for line in env_lines + if "=" in line + } + os.environ.update(env_dict) + + return source_syntax, and_sign diff --git a/eburger/utils/installers.py b/eburger/utils/installers.py index 3969ef0..4e12cf2 100644 --- a/eburger/utils/installers.py +++ b/eburger/utils/installers.py @@ -1,6 +1,6 @@ import os from eburger import settings -from eburger.utils.helpers import run_command +from eburger.utils.helpers import python_shell_source, run_command from eburger.utils.logger import log @@ -14,10 +14,8 @@ def install_foundry_if_not_found(): if not forge_binary_found: log("info", "forge wasn't found on the system, trying to install.") - run_command( - "curl -L https://foundry.paradigm.xyz | bash", - shell=True, - ) + run_command("curl -L https://foundry.paradigm.xyz | bash", shell=True) + python_shell_source() run_command("foundryup") try: run_command("forge -V") @@ -47,14 +45,54 @@ def install_hardhat_if_not_found(): run_command("npm -v") npm_found = True except FileNotFoundError: - log( - "error", - "Can't automatically install hardhat without npm being installed manually first, please install npm and run again.", + run_command( + "curl -L https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash", + shell=True, ) + # NVM + try: + _, errs = run_command( + construct_sourceable_nvm_string("nvm --version"), + shell=True, + live_output=True, + ) + if errs: + raise Exception + log("info", "Successfully installed nvm.") + except Exception as e: + print(e) + log( + "error", + "Couldn't automatically install nvm, please install manually and try again.", + ) + + # nodejs + try: + run_command( + construct_sourceable_nvm_string("nvm install --lts"), + shell=True, + live_output=True, + ) + run_command( + construct_sourceable_nvm_string("npm -v"), + shell=True, + live_output=True, + ) + npm_found = True + except FileNotFoundError: + log( + "error", + "Couldn't automatically install npm, please install manually and try again.", + ) + if npm_found: try: - run_command("npx -v") + run_command( + construct_sourceable_nvm_string("npx -v"), + shell=True, + live_output=True, + ) npx_found = True except FileNotFoundError: log( @@ -64,8 +102,21 @@ def install_hardhat_if_not_found(): if not npx_found: try: - run_command("npm install -g npx") - run_command("npx -v") + run_command( + construct_sourceable_nvm_string("npm install -g npx"), + shell=True, + live_output=True, + ) + run_command( + construct_sourceable_nvm_string("npm install -g yarn"), + shell=True, + live_output=True, + ) + run_command( + construct_sourceable_nvm_string("npx -v"), + shell=True, + live_output=True, + ) except FileNotFoundError: log( "error", @@ -75,12 +126,14 @@ def install_hardhat_if_not_found(): try: if os.path.isfile(os.path.join(settings.project_root, "yarn.lock")): run_command( - "yarn add --dev hardhat", + construct_sourceable_nvm_string("yarn add --dev hardhat"), directory=settings.project_root, ) else: run_command( - "npm install --save-dev hardhat", + construct_sourceable_nvm_string( + "npm install --save-dev hardhat" + ), directory=settings.project_root, ) except: @@ -103,3 +156,10 @@ def set_solc_compiler_version(solc_required_version: str): "info", "Successfully set solc version, trying to compile contract.", ) + + +# Used to prepare inputs for run_command for the python subprocess could reach nvm and its installed binaries, +# without the user having to reload the terminal. +def construct_sourceable_nvm_string(nvm_command: str) -> str: + source_syntax, and_sign = python_shell_source(execute_source=False) + return f'/bin/bash -c "{source_syntax} ~/.nvm/nvm.sh {and_sign} {nvm_command}"' diff --git a/eburger/yaml_parser.py b/eburger/yaml_parser.py index ffcdc32..19e0ecd 100644 --- a/eburger/yaml_parser.py +++ b/eburger/yaml_parser.py @@ -1,11 +1,11 @@ import traceback import yaml import concurrent.futures -import ast from eburger import settings from eburger.template_utils import * from eburger.utils.logger import color, log +from eburger.utils.cli_args import args def execute_python_code( @@ -76,6 +76,9 @@ def process_files_concurrently(ast_data: dict, src_file_list: list) -> list: try: results = future.result(timeout=30) # 30 seconds timeout if results.get("results"): + if args.no: + if results.get("severity").casefold() in args.no: + continue insights.append(results) except concurrent.futures.TimeoutError: log("error", "A task has timed out.") diff --git a/tests/test_arguments.py b/tests/test_arguments.py index b6b1f88..2edbd1f 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -18,11 +18,12 @@ def test_arguments(): args.solidity_file_or_folder = single_contract_path main() - # SARIF, Markdown + # SARIF args.output = "sarif" main() - args.output = "markdown" + # Markdown + custom file output path + args.output = "../eburger-output.md" main() # Single traversed file