From 06a054d5f1cc48e2a454beb90b92edc371ce9e67 Mon Sep 17 00:00:00 2001 From: Joel Winarske Date: Tue, 4 Feb 2025 09:26:57 -0800 Subject: [PATCH] MacOS sudo free Signed-off-by: Joel Winarske --- ' | 19 ++ .github/workflows/macos.yaml | 57 ++++++ .../{workspace-automation.yml => ubuntu.yml} | 0 common.py | 45 +++++ configs/toolchain-llvm-18.json | 16 +- flutter_workspace.py | 180 +++++++----------- 6 files changed, 204 insertions(+), 113 deletions(-) create mode 100644 ' create mode 100644 .github/workflows/macos.yaml rename .github/workflows/{workspace-automation.yml => ubuntu.yml} (100%) diff --git a/' b/' new file mode 100644 index 0000000..3b98f19 --- /dev/null +++ b/' @@ -0,0 +1,19 @@ +MacOS sudo free + +Signed-off-by: Joel Winarske + +# Please enter the commit message for your changes. Lines starting +# with '#' will be ignored, and an empty message aborts the commit. +# +# Date: Tue Feb 4 09:26:57 2025 -0800 +# +# On branch jw/macos +# Your branch is up to date with 'origin/jw/macos'. +# +# Changes to be committed: +# new file: .github/workflows/macos.yaml +# renamed: .github/workflows/workspace-automation.yml -> .github/workflows/ubuntu.yml +# modified: common.py +# modified: configs/toolchain-llvm-18.json +# modified: flutter_workspace.py +# diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml new file mode 100644 index 0000000..7ece7f2 --- /dev/null +++ b/.github/workflows/macos.yaml @@ -0,0 +1,57 @@ +name: macos-workspace-automation + +on: + pull_request: + types: [ opened, synchronize, reopened, closed ] + release: + types: [ published, created, edited ] + workflow_dispatch: + schedule: + # daily + - cron: '0 0 * * *' + +jobs: + + workspace-automation: + + env: + NONINTERACTIVE: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + + strategy: + matrix: + os: [ macos-14, macos-15 ] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Brew fixup + run: | + brew uninstall ruby@3.0 | true + brew unlink ruby@3.0 | true + brew uninstall openssl@1.1 | true + brew link --overwrite openssl@3 + + brew untap homebrew/cask | true + brew untap homebrew/core | true + + brew install ruby | true + brew update + brew upgrade + brew cleanup |true + brew doctor |true + ruby -v + + - name: Create workspace + run: | + ./flutter_workspace.py + + - name: Test workspace + run: | + source ./setup_env.sh + flutter doctor -v + dart --version diff --git a/.github/workflows/workspace-automation.yml b/.github/workflows/ubuntu.yml similarity index 100% rename from .github/workflows/workspace-automation.yml rename to .github/workflows/ubuntu.yml diff --git a/common.py b/common.py index be56af7..efdbd03 100755 --- a/common.py +++ b/common.py @@ -9,7 +9,9 @@ import errno import os import sys +import subprocess +from platform import system from sys import stderr as stream # use kiB's @@ -261,6 +263,49 @@ def fetch_https_binary_file(url, filename, redirect, headers, cookie_file, netrc return success +def get_ws_folder(): + if "FLUTTER_WORKSPACE" in os.environ: + workspace = os.environ.get('FLUTTER_WORKSPACE') + else: + workspace = os.getcwd() + return workspace + + +def get_host_type() -> str: + """returns host type in lower case""" + return system().lower().rstrip() + + +def reset_sudo_timestamp(): + """invalidate sudo timestamp file""" + if get_host_type() == "linux": + subprocess.check_call(['sudo', '-k'], stdout=subprocess.DEVNULL) + + +def validate_sudo_user_timestamp(args): + """read password from standard input if available""" + if os.path.exists(args.stdin_file): + stdin_file = open(args.stdin_file) + if get_host_type() == "linux": + subprocess.check_call(['sudo', '-S', '-v'], stdout=subprocess.DEVNULL, stdin=stdin_file) + else: + if get_host_type() == "linux": + subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + + +def validate_sudo_user(): + """update user's sudo timestamp without running a command""" + if get_host_type() == "linux": + subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + + +def chown_workspace(username, workspace): + """chown workspace if linux""" + if get_host_type() == "linux": + cmd = ['sudo', 'chown', '-R', username, workspace] + subprocess.check_call(cmd, cwd=workspace, stdout=subprocess.DEVNULL) + + def test_internet_connection() -> bool: """Test internet by connecting to nameserver""" import pycurl diff --git a/configs/toolchain-llvm-18.json b/configs/toolchain-llvm-18.json index 106f667..d03a6ce 100644 --- a/configs/toolchain-llvm-18.json +++ b/configs/toolchain-llvm-18.json @@ -3,17 +3,20 @@ "load": true, "supported_archs": [ "aarch64", + "arm64", "x86_64" ], "supported_host_types": [ - "ubuntu", - "fedora" + "darwin", + "fedora", + "ubuntu" ], "type": "toolchain", "toolchain": "llvm", "prefer_llvm": "18", "env": { - "VERSION": "18" + "VERSION": "18", + "NONINTERACTIVE": "1" }, "runtime": { "pre-requisites": { @@ -34,6 +37,13 @@ ] } }, + "arm64": { + "darwin": { + "cmds": [ + "brew install llvm" + ] + } + }, "x86_64": { "ubuntu": { "cmds": [ diff --git a/flutter_workspace.py b/flutter_workspace.py index 5be3e63..1916d21 100755 --- a/flutter_workspace.py +++ b/flutter_workspace.py @@ -48,6 +48,11 @@ from common import handle_ctrl_c from common import make_sure_path_exists from common import print_banner +from common import get_ws_folder +from common import reset_sudo_timestamp +from common import validate_sudo_user_timestamp +from common import validate_sudo_user +from common import chown_workspace from create_aot import get_flutter_sdk_version from create_aot import create_platform_aot @@ -142,6 +147,11 @@ def main(): flutter_analyze_git_commits() return + # + # Control+C handler + # + signal.signal(signal.SIGINT, handle_ctrl_c) + user = get_process_stdout('whoami').split('\n') username = user[0] print_banner("Running as: %s" % username) @@ -150,25 +160,17 @@ def main(): print("Please run as non-root user") exit() - # reset sudo timestamp - subprocess.check_call(['sudo', '-k'], stdout=subprocess.DEVNULL) + # + # Handle sudo for scenarios that require it + # + reset_sudo_timestamp() - # validate sudo user timestamp - if os.path.exists(args.stdin_file): - stdin_file = open(args.stdin_file) - subprocess.check_call(['sudo', '-S', '-v'], - stdout=subprocess.DEVNULL, stdin=stdin_file) - else: - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user_timestamp(args) # # Target Folder # - if "FLUTTER_WORKSPACE" in os.environ: - workspace = os.environ.get('FLUTTER_WORKSPACE') - else: - workspace = os.getcwd() - + workspace = get_ws_folder() config_folder = os.path.join(workspace, '.config') print_banner("Setting up Flutter Workspace in: %s" % workspace) @@ -176,32 +178,17 @@ def main(): # # Recursively change ownership to logged in user # - cmd = ['sudo', 'chown', '-R', username, workspace] - subprocess.check_call(cmd, stdout=subprocess.DEVNULL) + chown_workspace(username, workspace) # # Install minimum package # install_minimum_runtime_deps() - # - # Virtual Python Setup - # - venv_dir = os.path.join(config_folder, 'venv') - subprocess.check_call([sys.executable, '-m', 'venv', venv_dir], stdout=subprocess.DEVNULL) - os.environ['PATH'] = '%s:%s' % (os.path.join(venv_dir, 'bin'), os.environ.get('PATH')) - - # - # Control+C handler - # - signal.signal(signal.SIGINT, handle_ctrl_c) - # # Create Workspace # - is_exist = os.path.exists(workspace) - if not is_exist: - os.makedirs(workspace) + make_sure_path_exists(workspace) if os.path.exists(workspace): os.environ['FLUTTER_WORKSPACE'] = workspace @@ -370,8 +357,7 @@ def main(): # # Recursively change ownership to logged in user # - cmd = ['sudo', 'chown', '-R', username, workspace] - subprocess.check_call(cmd, cwd=workspace) + chown_workspace(username, workspace) # # Done @@ -686,15 +672,15 @@ def get_workspace_repos(base_folder, config): for repo in repos: futures.append(executor.submit(get_repo, base_folder=base_folder, uri=repo.get( 'uri'), branch=repo.get('branch'), rev=repo.get('rev'))) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() for _ in concurrent.futures.as_completed(futures): - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() print_banner("Repos Cloned") # reset sudo timeout - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() # # Create vscode startup tasks @@ -721,16 +707,15 @@ def get_platform_src(src, base_folder: str): for repo in src: futures.append(executor.submit(get_repo, base_folder=base_folder, uri=repo.get( 'uri'), branch=repo.get('branch'), rev=repo.get('rev'))) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() for future in concurrent.futures.as_completed(futures): future.result() # Get result to propagate any exceptions - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() print_banner("Source Repos Cloned") - # reset sudo timeout - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() def get_flutter_settings_folder(): @@ -1391,13 +1376,11 @@ def handle_http_obj(obj, host_machine_arch, cwd, cookie_file, netrc): futures.append(executor.submit(download_https_file, cwd, base_url, filename, cookie_file, netrc, artifact.get('md5'), artifact.get('sha1'), artifact.get('sha256'), True)) - subprocess.check_call( - ['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() for future in concurrent.futures.as_completed(futures): future.result() - subprocess.check_call( - ['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() def handle_commands(cmds, cwd): @@ -1823,26 +1806,26 @@ def setup_platform(platform_, git_token, cookie_file, plex, enable, disable, app cwd = get_platform_working_dir(platform_['id']) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_dotenv(platform_.get('dotenv')) handle_env(platform_.get('env'), None) create_platform_config_file(runtime.get('config'), cwd) create_gclient_config_file(runtime.get('gclient_config')) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_artifacts_obj(runtime.get('artifacts'), host_machine_arch, cwd, git_token, cookie_file) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_pre_requisites(runtime.get('pre-requisites'), cwd) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_docker_obj(runtime.get('docker'), host_machine_arch, cwd) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_conditionals(runtime.get('conditionals'), cwd) - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_qemu_obj(runtime.get('qemu'), cwd, platform_[ 'id'], 'debug') - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() handle_commands_obj(runtime.get('post_cmds'), cwd) handle_custom_devices(platform_) @@ -1943,7 +1926,15 @@ def setup_toolchain(platform_, git_token, cookie_file, plex, enable, disable, ap os.environ['PREFER_LLVM'] = prefer_llvm print(f'PREFER_LLVM: {prefer_llvm}') - llvm_config = find_llvm_config_in_sysroot('/usr', prefer_llvm) + host_type = get_host_type() + + if host_type == 'linux': + llvm_base_path = '/usr' + elif host_type == 'darwin': + prefer_llvm = None + llvm_base_path = '/opt/homebrew' + + llvm_config = find_llvm_config_in_sysroot(llvm_base_path, prefer_llvm) if llvm_config: llvm_prefix = get_llvm_prefix(llvm_config) llvm_version = get_llvm_version(llvm_config) @@ -1966,7 +1957,7 @@ def setup_toolchain(platform_, git_token, cookie_file, plex, enable, disable, ap print(f"CC: {os.environ['CC']}") print(f"CXX: {os.environ['CXX']}") else: - print_banner("Failed to find llvm-config in /usr.") + print_banner(f'Failed to find llvm-config in {llvm_base_path}.') exit(1) elif platform_['toolchain'] == 'common': @@ -2022,14 +2013,14 @@ def setup_platforms(platforms, git_token, cookie_file, plex, enable, disable, ap setup_platform(dependency, git_token, cookie_file, plex, enable, disable, app_folder) # reset sudo timeout - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() # iterate over non-dependencies for platform_ in get_not_dependencies(platforms): setup_platform(platform_, git_token, cookie_file, plex, enable, disable, app_folder) # reset sudo timeout - subprocess.check_call(['sudo', '-v'], stdout=subprocess.DEVNULL) + validate_sudo_user() print_banner("Platform Setup Complete") @@ -2201,62 +2192,19 @@ def get_mac_brew_path() -> str: return result.stdout.decode('utf-8').rstrip() -def get_mac_openssl_prefix() -> str: - """ Read brew openssl prefix variable """ - if platform.machine() == 'arm64': - return subprocess.check_output( - ['arch', '-arm64', 'brew', '--prefix', 'openssl@3']).decode('utf-8').rstrip() - else: - return subprocess.check_output(['brew', '--prefix', 'openssl@3']).decode('utf-8').rstrip() - - -def mac_brew_reinstall_package(pkg): - """ Re-installs brew package """ - if platform.machine() == 'arm64': - subprocess.run(['arch', '-arm64', 'brew', 'reinstall', pkg]) - else: - subprocess.run(['brew', 'reinstall', pkg]) - - -def mac_pip3_install(pkg): - """ Install pip3 on mac """ - if platform.machine() == 'arm64': - cmd = 'arch -arm64 pip3 install %s' % pkg - else: - cmd = 'pip3 install %s' % pkg - p = subprocess.Popen(cmd, universal_newlines=True, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - text = p.stdout.read() - p.wait() - print(text) - - -def mac_is_cocoapods_installed(): - cmd = ['gem', 'list', '|', 'grep', 'cocoapods '] - - result = get_process_stdout(cmd).strip('\'').strip('\n') - - if 'cocoapods ' in result: - print("Package cocoapods Found") - return True - else: - print("Package cocoapods Not Found") - return False - - -def mac_install_cocoapods_if_not_installed(): - if not mac_is_cocoapods_installed(): - subprocess.check_output( - ['sudo', 'gem', '-y', 'install', 'activesupport', '-v', '6.1.7.3']) - subprocess.check_output(['sudo', 'gem', '-y', 'install', 'cocoapods']) - subprocess.run( - ['sudo', 'gem', 'uninstall', 'ffi', '&&', 'sudo', 'gem', 'install', 'ffi', '--', '--enable-libffi-alloc']) +def activate_python_venv(): + workspace = get_ws_folder() + config_folder = os.path.join(workspace, '.config') + venv_dir = os.path.join(config_folder, 'venv') + subprocess.check_call([sys.executable, '-m', 'venv', venv_dir], stdout=subprocess.DEVNULL) + os.environ['PATH'] = '%s:%s' % (os.path.join(venv_dir, 'bin'), os.environ.get('PATH')) def install_minimum_runtime_deps(): """Install minimum runtime deps to run this script""" host_type = get_host_type() + if host_type == "linux": os_release_id = get_freedesktop_os_release_id() @@ -2271,18 +2219,30 @@ def install_minimum_runtime_deps(): packages = 'sudo dnf -y install dnf-plugins-core git git-lfs unzip curl python3-pip libcurl-devel openssl-devel gtk3-devel python3-virtualenv python3-pycurl python3-toml python3-dotenv python3-devel gcc libcurl-devel'.split(' ') subprocess.check_output(packages) - elif host_type == "darwin": + activate_python_venv() + + + if host_type == "darwin": + brew_path = get_mac_brew_path() if brew_path == '': sys.exit( "brew is required for this script. Please install. https://brew.sh") - mac_brew_reinstall_package('openssl@3') + os.environ['NONINTERACTIVE'] = '1' + subprocess.check_output(['brew', 'update']) + subprocess.check_output(['brew', 'doctor']) + + packages = 'brew install git git-lfs unzip curl python3'.split(' ') + subprocess.check_output(packages) + + activate_python_venv() - mac_pip3_install( - '--install-option="--with-openssl" --install-option="--openssl-dir=%s" pycurl' % (get_mac_openssl_prefix())) + cmd = 'pip install --upgrade pip'.split(' ') + subprocess.check_output(cmd) - mac_install_cocoapods_if_not_installed() + cmd = 'python -m pip install toml pycurl python-dotenv'.split(' ') + subprocess.check_output(cmd) def is_repo(path):