From 21ca95558d944f1224283ac426c2facdbf4708f0 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Fri, 20 Dec 2024 21:25:21 +0530 Subject: [PATCH 01/21] Add launcher.py --- launcher.py | 479 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 launcher.py diff --git a/launcher.py b/launcher.py new file mode 100644 index 0000000..74eec44 --- /dev/null +++ b/launcher.py @@ -0,0 +1,479 @@ +import argparse +import atexit +import json +import logging +import logging.config +import os +import platform +import subprocess +import sys +import threading +import time +import webbrowser +from pathlib import Path + +CONFIG_FILE = os.path.join("config.json") +venv_name = "virtual_env" # For downloading python packages +venv_dir = Path.cwd() / venv_name # Generate the path for the virtual environment +DEFAULT_CONFIG = { + "version": 1, + "formatters": { + "detailed": {"format": "%(asctime)s [%(levelname)s] [%(funcName)s(), %(lineno)d]: %(message)s"}, + "simple": {"format": "[%(levelname)s] %(message)s"}, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple", + "stream": "ext://sys.stdout", + }, + }, + "root": {"level": "ERROR", "handlers": ["console"]}, +} + + +def setup_logging(log_level, log_file=False): + DEFAULT_CONFIG["root"]["level"] = log_level + if log_file: + DEFAULT_CONFIG["handlers"]["file"] = { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "detailed", + "filename": f"logfile_{time.strftime('%Y%m%d_%H%M%S')}.log", + "maxBytes": 1024 * 1024 * 5, # 5MB + "backupCount": 3, + } + DEFAULT_CONFIG["root"]["handlers"].append("file") + + logging.config.dictConfig(DEFAULT_CONFIG) + + +def run_command(command, cwd=None): + """ + Executes a shell command. + + Args: + command (str): The command to run. + + Raises: + subprocess.CalledProcessError: If any command fails. + """ + logging.debug(f"Executing command: {command}") + process = subprocess.Popen( + command.split(), + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = process.communicate() # wait for process to terminate + if stdout: + logging.debug(f"Command output: {stdout}") + if stderr: + logging.error(f"Command error: {stderr}") + + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command) + + +class DependencyHandler: + """ + Represents a dependency with methods for checking and installing it. + + Attributes: + name (str): The name of the dependency. + package_name (dict): { platform: package_name }. + check_cmd (dict): { platform: list of command or function to check if the dependency is installed }. + install_cmd (dict): { platform: list of command or function to install the dependency }. + + if package_name is not provided, it will be set to the name. + + if check_cmd is not provided + package_name --version will be used to check if the dependency is installed + + if install cmd is not provided + for linux: package manager based on distro + for macos: brew + for windows: msys2 + """ + + def __init__(self, name, package_name=None, check_cmd=None, install_cmd=None): + self.name = name + self.package_name = package_name or {} + self.check_cmd = check_cmd or {} + self.install_cmd = install_cmd or {} + self._installers = { + "Linux": self._get_install_commands_linux, + "Darwin": self._get_install_commands_darwin, + "Windows": self._get_install_commands_windows, + } + # Mapping of Linux distributions to package managers + self._linux_distro_map = { + "ubuntu": "sudo apt-get install -y", + "debian": "sudo apt-get install -y", + "kali": "sudo apt-get install -y ", + "pop": "sudo apt-get install -y ", + "elementary": "sudo apt-get install -y ", + "mint": "sudo apt-get install -y", + "fedora": "sudo dnf install -y", + "rhel": "sudo dnf install -y", + "centos": "sudo dnf install -y", + "rocky": "sudo dnf install -y", + "alma": "sudo dnf install -y", + "arch": "sudo pacman -S --needed --noconfirm", + "manjaro": "sudo pacman -S --needed --noconfirm", + "endeavouros": "sudo pacman -S --needed --noconfirm", + "garuda": "sudo pacman -S --needed --noconfirm", + "opensuse": "sudo zypper install -y", + "suse": "sudo zypper install -y", + "alpine": "sudo apk add", + "solus": "sudo eopkg install -y", + "void": "sudo xbps-install -y", + "clearlinux": "sudo swupd bundle-add", + } + + def ensure_dependency_installed(self, system, install=True): + """ + Checks if the dependency is installed. If not installs it if install arg is True. + """ + commands = self._get_check_commands(system) + try: + for cmd in commands: + if type(cmd) == str: + run_command(cmd) + else: + cmd() + + except (FileNotFoundError, subprocess.CalledProcessError): + if not install: + logging.info(f"{self.name} is not installed.") + sys.exit(1) + self.install_dependency(system) + + except Exception as e: + logging.error(f"While Checking Dependency: {e}", exc_info=True) + sys.exit(1) + + def install_dependency(self, system): + """ + Installs the dependency for the specified operating system. + + Args: + system (str): The operating system type. + """ + try: + commands = self._get_install_commands(system) + for cmd in commands: + if type(cmd) == str: + run_command(cmd) + else: + cmd() + except Exception as e: + logging.error(f"While Installing: {e}.", exc_info=True) + sys.exit(1) + + def _get_check_commands(self, system): + if system in self.check_cmd: + return self.check_cmd[system] + if "all" in self.check_cmd: + return self.check_cmd["all"] + return [f"{self.package_name.get(system, self.name)} --version"] + + def _get_install_commands(self, system): + if system in self.install_cmd: + return self.install_cmd[system] + if "all" in self.install_cmd: + return self.install_cmd["all"] + if system not in self._installers: + logging.error(f"Unspported sytem {system}.") + sys.exit(1) + + return self._installers[system]() + + def _get_install_commands_linux(self): + import distro + + distro = distro.id().lower() + if distro not in self._linux_distro_map: + logging.error(f"Unsupported linux distro {distro}.") + sys.exit(1) + + return [f"{self._linux_distro_map[distro]} {self.package_name.get('Linux', self.name)}"] + + def _get_install_commands_darwin(self): + return [f"brew install {self.package_name.get('Darwin', self.name)}"] + + def _get_install_commands_windows(self): + if not check_msys2_installed(): + install_msys2() + return [f"pacman -S --needed --noconfirm {self.package_name.get('Windows', self.name)}"] + + +# pkg-config should come before sndfie and nlohmann-json +dependencies = [ + DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), + DependencyHandler( + "g++", + {"Windows": "mingw-w64-x86_64-gcc", "Darwin": "gcc"}, + {"all": ["g++ --version"]}, + {"Windows": ["pacman -S --needed --noconfirm base-devel mingw-w64-x86_64-toolchain"]}, + ), + DependencyHandler("pkg-config"), + DependencyHandler("ffmpeg", {"Windows": "mingw-w64-x86_64-ffmpeg"}, {"all": ["ffmpeg -version"]}), + DependencyHandler( + "libsndfile", + {"Windows": "mingw-w64-x86_64-libsndfile", "Linux": "libsndfile1-dev"}, + {"all": ["pkg-config --exists sndfile"]}, + ), + DependencyHandler( + "nlohmann-json", + {"Windows": "mingw-w64-x86_64-nlohmann-json", "Linux": "nlohmann-json3-dev"}, + {"all": ["pkg-config --exists nlohmann_json"]}, + ), +] + + +def check_msys2_installed(): + try: + run_command("pacman --version") + return True + except (FileNotFoundError, subprocess.CalledProcessError): + return False + + +def check_internet_connectivity(): + """For Windows check internet connectivity""" + try: + logging.debug("Checking internet connectivity... ") + # Ping Google's public DNS server + run_command("ping -n 1 8.8.8.8") + logging.debug("Internet connectivity OK") + except subprocess.CalledProcessError: + logging.error("No internet connection detected.") + sys.exit(1) + + +def install_msys2(): + try: + installer_url = ( + "https://github.com/msys2/msys2-installer/releases/download/nightly-x86_64/msys2-base-x86_64-latest.sfx.exe" + ) + installer_name = "msys2-installer.exe" + + msys2_root_path = "C:\\msys64" + + check_internet_connectivity() + + logging.info("Downloading MSYS2 installer...") + logging.debug(f"Installer URL: {installer_url}") + run_command(f"curl -L -o {installer_name} {installer_url}") + + logging.info("Running MSYS2 installer...") + logging.debug(f"Installing MSYS2 at {msys2_root_path}") + run_command(f"{installer_name} -y -oC:\\") + + logging.info("Updating MSYS2 packages...") + run_command(f"{msys2_root_path}\\usr\\bin\\bash.exe -lc 'pacman -Syu --noconfirm'") + + logging.info("Editing Environment Variables...") + logging.debug( + f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to PATH" + ) + # Set it permanently for the current user + commands = f""" + $oldPath = [Environment]::GetEnvironmentVariable("Path", "User") + $newPath = "{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" + $oldPath + [Environment]::SetEnvironmentVariable("Path", $newPath, "User") + """ + # Run the PowerShell commands + subprocess.check_call(["powershell", "-Command", commands]) + + # Add MSYS2 paths to the PATH environment variable for the current session + logging.debug( + f"Addiing {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to current enviornment." + ) + current_path = os.environ.get("PATH", "") + new_path = ( + f"{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" + + current_path + ) + os.environ["PATH"] = new_path + + logging.info("MSYS2 installed and updated successfully.") + logging.info("NOTE: Please restart your terminal before running this script again.") + + except subprocess.CalledProcessError as e: + logging.error(f"Error installing MSYS2: {e}", exc_info=True) + sys.exit(1) + finally: + if os.path.exists(installer_name): + os.remove(installer_name) + + +def check_MediaProcessor(system): + if system == "Windows": + MediaProcessor_path = Path("MediaProcessor") / "build" / "MediaProcessor.exe" + else: + MediaProcessor_path = Path("MediaProcessor") / "build" / "MediaProcessor" + return MediaProcessor_path.exists() + + +def generate_virtualenv(): + logging.info(f"Generating virtual environment at: {venv_dir}") + run_command(f"{sys.executable} -m venv {str(venv_dir)} --system-site-packages") + logging.info("Successfully generated Virtual environment.") + + +def install_python_dependencies(system): + try: + if not venv_dir.exists(): + generate_virtualenv() + + logging.info("Installing Python dependencies...") + + if system == "Windows": + # Check for both `Scripts` and `bin` folders. + pip_scripts_path = venv_dir / "Scripts" / "pip.exe" + pip_bin_path = venv_dir / "bin" / "pip.exe" + + if not pip_scripts_path.exists() and not pip_bin_path.exists(): + logging.error(f"pip not found. Searched paths: [{pip_scripts_path}] and [{pip_bin_path}]") + sys.exit(1) + + if pip_bin_path.exists(): + pip_path = pip_bin_path + else: + pip_path = pip_scripts_path + + else: + pip_path = venv_dir / "bin" / "pip" + if not pip_path.exists(): + logging.error(f"pip not found. Searched paths: [{pip_path}]") + sys.exit(1) + + run_command(f"{str(pip_path)} install -r requirements.txt") + logging.info("Python dependencies installed.") + except Exception as e: + logging.error(f"Failed to install Python dependencies: {e}", exc_info=True) + sys.exit(1) + + +def build_cpp_dependencies(): + try: + logging.info("Building MediaProcessor...") + build_dir = os.path.join("MediaProcessor", "build") + if not os.path.exists(build_dir): + os.makedirs(build_dir) + + run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=build_dir) + run_command("cmake --build . --config Release", cwd=build_dir) + + logging.info("MediaProcessor built successfully.") + except Exception as e: + logging.error(f"Failed to build MediaProcessor: {e}", exc_info=True) + sys.exit(1) + + +def update_config(system): + try: + with open(CONFIG_FILE, "r") as config_file: + config = json.load(config_file) + + if system == "Windows": + for key, value in config.items(): + if type(value) == str: + config[key] = value.replace("/", "\\") + + with open(CONFIG_FILE, "w") as config_file: + json.dump(config, config_file, indent=4) + + except FileNotFoundError: + logging.error(f"{CONFIG_FILE} not found. Please create a default config.json file.") + sys.exit(1) + except Exception as e: + logging.error(f"Failed to update config: {e}", exc_info=True) + sys.exit(1) + + +def log_stream(stream, log_function): + """Logs output from a stream.""" + for line in iter(stream.readline, ""): + log_function(line.strip()) + stream.close() + + +def launch_web_application(system): + try: + if not venv_dir.exists(): + generate_virtualenv() + + if system == "Windows": + python_path = str(venv_dir / "Scripts" / "python.exe") + else: + python_path = str(venv_dir / "bin" / "python") + + # Start the backend + app_process = subprocess.Popen( + [python_path, "app.py"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + atexit.register(app_process.terminate) + + # Threads to handle stdout and stderr asynchronously + threading.Thread(target=log_stream, args=(app_process.stdout, logging.debug), daemon=True).start() + threading.Thread(target=log_stream, args=(app_process.stderr, logging.debug), daemon=True).start() + + # Give the process some time to initialize + time.sleep(0.5) + + # Check if the process is still running + if app_process.poll() is not None: + error_output = app_process.stderr.read() + logging.error(f"Error starting the backend: {error_output}") + sys.exit(1) + + webbrowser.open("http://127.0.0.1:8080") + + logging.info("Web application running. Press Enter to stop.") + input() # Block until the user presses Enter + + except Exception as e: + logging.error(f"An error occurred: {e}", exc_info=True) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser(description="Setup for MediaProcessor Application") + parser.add_argument("--app", choices=["web", "none"], default="web", help="Specify launch mode") + parser.add_argument("--install-dependencies", action="store_true", default=True, help="Install dependencies.") + parser.add_argument("--rebuild", action="store_true", help="Rebuild MediaProcessor") + parser.add_argument( + "--debug-level", choices=["DEBUG", "INFO", "ERROR"], default="INFO", help="Set the debug output level." + ) + parser.add_argument("--debug-file", action="store_true", help="Set the debug file.") + args = parser.parse_args() + + system = platform.system() + setup_logging(args.debug_level, args.debug_file) + logging.info("Starting setup...") + + for dependency in dependencies: + dependency.ensure_dependency_installed(system, args.install_dependencies) + + if args.install_dependencies: + install_python_dependencies(system) + + if args.rebuild or not check_MediaProcessor(system): + build_cpp_dependencies() + + update_config(system) + + if args.app == "web": + launch_web_application(system) + else: + logging.info("Please specify how you would like to launch the application, like --app=web.") + + +if __name__ == "__main__": + main() From fa3f7a6ef607a952fa86d007fa23a0777214f97d Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 10:18:38 +0530 Subject: [PATCH 02/21] Refactor launcher.py: 1. Capitalize virutal env constants `VENV_NAME` and `VENV_DIR` 2. Use pathlib lib for paths --- launcher.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/launcher.py b/launcher.py index 74eec44..2f6c721 100644 --- a/launcher.py +++ b/launcher.py @@ -12,9 +12,9 @@ import webbrowser from pathlib import Path -CONFIG_FILE = os.path.join("config.json") -venv_name = "virtual_env" # For downloading python packages -venv_dir = Path.cwd() / venv_name # Generate the path for the virtual environment +CONFIG_FILE = Path("config.json") +VENV_NAME = "virtual_env" # For downloading python packages +VENV_DIR = Path.cwd() / VENV_NAME # Generate the path for the virtual environment DEFAULT_CONFIG = { "version": 1, "formatters": { @@ -53,6 +53,7 @@ def run_command(command, cwd=None): Args: command (str): The command to run. + cwd (str, optional): The working directory. Defaults to None. Raises: subprocess.CalledProcessError: If any command fails. @@ -305,7 +306,7 @@ def install_msys2(): logging.error(f"Error installing MSYS2: {e}", exc_info=True) sys.exit(1) finally: - if os.path.exists(installer_name): + if Path.exists(installer_name): os.remove(installer_name) @@ -318,22 +319,22 @@ def check_MediaProcessor(system): def generate_virtualenv(): - logging.info(f"Generating virtual environment at: {venv_dir}") - run_command(f"{sys.executable} -m venv {str(venv_dir)} --system-site-packages") + logging.info(f"Generating virtual environment at: {VENV_DIR}") + run_command(f"{sys.executable} -m venv {str(VENV_DIR)} --system-site-packages") logging.info("Successfully generated Virtual environment.") def install_python_dependencies(system): try: - if not venv_dir.exists(): + if not VENV_DIR.exists(): generate_virtualenv() logging.info("Installing Python dependencies...") if system == "Windows": # Check for both `Scripts` and `bin` folders. - pip_scripts_path = venv_dir / "Scripts" / "pip.exe" - pip_bin_path = venv_dir / "bin" / "pip.exe" + pip_scripts_path = VENV_DIR / "Scripts" / "pip.exe" + pip_bin_path = VENV_DIR / "bin" / "pip.exe" if not pip_scripts_path.exists() and not pip_bin_path.exists(): logging.error(f"pip not found. Searched paths: [{pip_scripts_path}] and [{pip_bin_path}]") @@ -345,7 +346,7 @@ def install_python_dependencies(system): pip_path = pip_scripts_path else: - pip_path = venv_dir / "bin" / "pip" + pip_path = VENV_DIR / "bin" / "pip" if not pip_path.exists(): logging.error(f"pip not found. Searched paths: [{pip_path}]") sys.exit(1) @@ -360,8 +361,8 @@ def install_python_dependencies(system): def build_cpp_dependencies(): try: logging.info("Building MediaProcessor...") - build_dir = os.path.join("MediaProcessor", "build") - if not os.path.exists(build_dir): + build_dir = Path.join("MediaProcessor", "build") + if not Path.exists(build_dir): os.makedirs(build_dir) run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=build_dir) @@ -403,13 +404,13 @@ def log_stream(stream, log_function): def launch_web_application(system): try: - if not venv_dir.exists(): + if not VENV_DIR.exists(): generate_virtualenv() if system == "Windows": - python_path = str(venv_dir / "Scripts" / "python.exe") + python_path = str(VENV_DIR / "Scripts" / "python.exe") else: - python_path = str(venv_dir / "bin" / "python") + python_path = str(VENV_DIR / "bin" / "python") # Start the backend app_process = subprocess.Popen( @@ -454,6 +455,11 @@ def main(): parser.add_argument("--debug-file", action="store_true", help="Set the debug file.") args = parser.parse_args() + """ + Initialise logging + check dependency + """ + system = platform.system() setup_logging(args.debug_level, args.debug_file) logging.info("Starting setup...") From f034b6c5854416294b9b9f65064ede984f065ce2 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 10:25:15 +0530 Subject: [PATCH 03/21] Remove unnessasary requirements from requirements.txt --- requirements.txt | 53 ++++++++---------------------------------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8651de8..f75c30d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,8 @@ -bidict==0.23.1 -black==24.10.0 -blinker==1.8.2 -Brotli==1.1.0 -certifi==2024.8.30 -charset-normalizer==3.3.2 -click==8.1.7 -dnspython==2.6.1 -eventlet==0.37.0 -ffmpeg-python==0.2.0 -flake8==7.1.1 -Flask==3.0.3 -Flask-Cors==5.0.0 -Flask-SocketIO==5.3.7 -future==1.0.0 -gevent==24.2.1 -gevent-websocket==0.10.1 -greenlet==3.1.0 -h11==0.14.0 -idna==3.8 -itsdangerous==2.2.0 -Jinja2==3.1.4 -MarkupSafe==2.1.5 -mccabe==0.7.0 -mutagen==1.47.0 -mypy-extensions==1.0.0 -packaging==24.1 -pathspec==0.12.1 -platformdirs==4.3.6 -pycodestyle==2.12.1 -pycryptodomex==3.20.0 -pyflakes==3.2.0 -python-engineio==4.9.1 -python-socketio==5.11.4 -requests==2.32.3 -simple-websocket==1.0.0 -tomli==2.0.2 -typing_extensions==4.12.2 -urllib3==2.2.2 -websockets==13.0.1 -Werkzeug==3.0.4 -wsproto==1.2.0 -yt-dlp==2024.9.27 -zope.event==5.0 -zope.interface==7.0.3 +blinker>=1.9.0 +click>=8.1.8 +Flask>=3.1.0 +itsdangerous>=2.2.0 +Jinja2>=3.1.5 +MarkupSafe>=3.0.2 +Werkzeug>=3.1.3 +yt-dlp>=2024.12.13 From 83ff05f8ecdb2ec22a080c26d8c7a8f948aa5be8 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 17:13:52 +0530 Subject: [PATCH 04/21] Refactor Dependency handling in launcher.py Add python dependencies and MSYS2 as dependency --- launcher.py | 357 ++++++++++++++++++++++++++++------------------------ 1 file changed, 191 insertions(+), 166 deletions(-) diff --git a/launcher.py b/launcher.py index 2f6c721..16047b0 100644 --- a/launcher.py +++ b/launcher.py @@ -10,11 +10,13 @@ import threading import time import webbrowser +from importlib.metadata import PackageNotFoundError, version from pathlib import Path CONFIG_FILE = Path("config.json") VENV_NAME = "virtual_env" # For downloading python packages VENV_DIR = Path.cwd() / VENV_NAME # Generate the path for the virtual environment +MEDIAPROCESSOR_PATH = Path("MediaProcessor") / "build" DEFAULT_CONFIG = { "version": 1, "formatters": { @@ -47,7 +49,7 @@ def setup_logging(log_level, log_file=False): logging.config.dictConfig(DEFAULT_CONFIG) -def run_command(command, cwd=None): +def run_command(command: str, cwd=None): """ Executes a shell command. @@ -76,6 +78,142 @@ def run_command(command, cwd=None): raise subprocess.CalledProcessError(process.returncode, command) +def check_internet_connectivity(system: str) -> bool: + """ + Check if the system has internet connectivity by pinging Google's public DNS server. + """ + try: + if system == "Windows": + cmd = "ping -n 1 8.8.8.8" + else: + cmd = "ping -c 1 8.8.8.8" + logging.debug("Checking internet connectivity... ") + run_command(cmd) + logging.debug("Internet connectivity OK") + return True + + except Exception as e: + logging.error("No internet connection detected.") + logging.debug(f"Error: {e}") + return False + + +def install_msys2(): + try: + installer_url = ( + "https://github.com/msys2/msys2-installer/releases/download/nightly-x86_64/msys2-base-x86_64-latest.sfx.exe" + ) + installer_name = "msys2-installer.exe" + + msys2_root_path = "C:\\msys64" + + logging.info("Downloading MSYS2 installer...") + logging.debug(f"Installer URL: {installer_url}") + run_command(f"curl -L -o {installer_name} {installer_url}") + + logging.info("Running MSYS2 installer...") + logging.debug(f"Installing MSYS2 at {msys2_root_path}") + run_command(f"{installer_name} -y -oC:\\") + + logging.info("Updating MSYS2 packages...") + run_command(f"{msys2_root_path}\\usr\\bin\\bash.exe -lc 'pacman -Syu --noconfirm'") + + logging.info("Editing Environment Variables...") + logging.debug( + f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to PATH" + ) + # Set it permanently for the current user + commands = f""" + $oldPath = [Environment]::GetEnvironmentVariable("Path", "User") + $newPath = "{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" + $oldPath + [Environment]::SetEnvironmentVariable("Path", $newPath, "User") + """ + # Run the PowerShell commands + subprocess.check_call(["powershell", "-Command", commands]) + + # Add MSYS2 paths to the PATH environment variable for the current session + logging.debug( + f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to current enviornment." + ) + current_path = os.environ.get("PATH", "") + new_path = ( + f"{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" + + current_path + ) + os.environ["PATH"] = new_path + + logging.info("MSYS2 installed and updated successfully.") + logging.info("NOTE: Please restart your terminal before running this script again.") + + except subprocess.CalledProcessError as e: + logging.error(f"Error installing MSYS2: {e}", exc_info=True) + sys.exit(1) + finally: + if Path(installer_name).exists(): + os.remove(installer_name) + + +def check_python_dependecies_installed(): + """ + check if the packages in the requirements.txt are installed + """ + try: + requirements = open("requirements.txt", "r").readlines() + for req in requirements: + req = req.strip() + if not req or req.startswith("#"): # Skip empty lines and comments + continue + operator = ">=" if ">=" in req else "==" + + package_name, required_version = req.split(operator) + package_name = package_name.strip() + installed_version = version(package_name) + + required_version = tuple(map(int, required_version.strip().split("."))) + installed_version = tuple(map(int, installed_version.strip().split("."))) + + logging.debug(f"installed version of {package_name}: {installed_version}") + if (operator == ">=" and installed_version >= required_version) or ( + operator == "==" and installed_version == required_version + ): + logging.debug(f"{package_name} is installed and meets the requirement.") + else: + logging.debug( + f"Version mismatch for {package_name}: " + f"installed {installed_version}, required {required_version}" + ) + raise FileNotFoundError + except (PackageNotFoundError, FileNotFoundError): + logging.debug(f"{package_name} is not installed.") + raise FileNotFoundError + except Exception as e: + logging.error(f"Error processing requirement {req}: {e}") + sys.exit(1) + + +def ensure_virtualenv_exists(): + if VENV_DIR.exists(): + logging.debug("Virtual environment already exists.") + return + logging.info(f"Generating virtual environment at: {VENV_DIR}") + run_command(f"{sys.executable} -m venv {str(VENV_DIR)} --system-site-packages") + logging.info("Successfully generated Virtual environment.") + + +def get_virutalenv_folder() -> Path: + """ + get the path of the virtual environment binaries folder + """ + ensure_virtualenv_exists() + for folder in ["bin", "Scripts"]: + path = VENV_DIR / folder + if path.exists(): + return path + + logging.error("Could not locate virtual environment folders.") + sys.exit(1) + + class DependencyHandler: """ Represents a dependency with methods for checking and installing it. @@ -95,6 +233,8 @@ class DependencyHandler: for linux: package manager based on distro for macos: brew for windows: msys2 + + Note: If a function is provided for check_cmd ensure it raise FileNotFoundError if the dependency is not installed. """ def __init__(self, name, package_name=None, check_cmd=None, install_cmd=None): @@ -132,10 +272,19 @@ def __init__(self, name, package_name=None, check_cmd=None, install_cmd=None): "clearlinux": "sudo swupd bundle-add", } - def ensure_dependency_installed(self, system, install=True): + def ensure_installed(self, system): + """ + Ensures the package is installed on the system. + """ + if self.check_installed(system): + return + self.install_dependency(system) + + def check_installed(self, system) -> bool: """ - Checks if the dependency is installed. If not installs it if install arg is True. + Checks if the dependency is installed. """ + logging.debug(f"Checking for {self.name} on {system}") commands = self._get_check_commands(system) try: for cmd in commands: @@ -143,18 +292,15 @@ def ensure_dependency_installed(self, system, install=True): run_command(cmd) else: cmd() - + return True except (FileNotFoundError, subprocess.CalledProcessError): - if not install: - logging.info(f"{self.name} is not installed.") - sys.exit(1) - self.install_dependency(system) - + logging.debug(f"{self.name} not Found.") + return False except Exception as e: logging.error(f"While Checking Dependency: {e}", exc_info=True) sys.exit(1) - def install_dependency(self, system): + def install_dependency(self, system: str): """ Installs the dependency for the specified operating system. @@ -162,12 +308,15 @@ def install_dependency(self, system): system (str): The operating system type. """ try: + check_internet_connectivity(system) + logging.debug(f"Installing {self.name} on {system}") commands = self._get_install_commands(system) for cmd in commands: if type(cmd) == str: run_command(cmd) else: cmd() + logging.debug(f"Successfully installed {self.name}") except Exception as e: logging.error(f"While Installing: {e}.", exc_info=True) sys.exit(1) @@ -204,13 +353,16 @@ def _get_install_commands_darwin(self): return [f"brew install {self.package_name.get('Darwin', self.name)}"] def _get_install_commands_windows(self): - if not check_msys2_installed(): - install_msys2() + MSYS2.ensure_installed() return [f"pacman -S --needed --noconfirm {self.package_name.get('Windows', self.name)}"] +MSYS2 = DependencyHandler( + "MSYS2", check_cmd={"Windows": ["pacman --version"]}, install_cmd={"Windows": [install_msys2]} +) + # pkg-config should come before sndfie and nlohmann-json -dependencies = [ +required_dependencies = [ DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), DependencyHandler( "g++", @@ -230,144 +382,32 @@ def _get_install_commands_windows(self): {"Windows": "mingw-w64-x86_64-nlohmann-json", "Linux": "nlohmann-json3-dev"}, {"all": ["pkg-config --exists nlohmann_json"]}, ), + DependencyHandler( + "Python Dependencies", + check_cmd={"all": [check_python_dependecies_installed]}, + install_cmd={ + "Windows": [f"{str(get_virutalenv_folder()/ "pip.exe")} install -r requirements.txt"], + "all": [f"{str(get_virutalenv_folder()/ "pip")} install -r requirements.txt"], + }, + ), ] -def check_msys2_installed(): - try: - run_command("pacman --version") - return True - except (FileNotFoundError, subprocess.CalledProcessError): - return False - - -def check_internet_connectivity(): - """For Windows check internet connectivity""" - try: - logging.debug("Checking internet connectivity... ") - # Ping Google's public DNS server - run_command("ping -n 1 8.8.8.8") - logging.debug("Internet connectivity OK") - except subprocess.CalledProcessError: - logging.error("No internet connection detected.") - sys.exit(1) - - -def install_msys2(): - try: - installer_url = ( - "https://github.com/msys2/msys2-installer/releases/download/nightly-x86_64/msys2-base-x86_64-latest.sfx.exe" - ) - installer_name = "msys2-installer.exe" - - msys2_root_path = "C:\\msys64" - - check_internet_connectivity() - - logging.info("Downloading MSYS2 installer...") - logging.debug(f"Installer URL: {installer_url}") - run_command(f"curl -L -o {installer_name} {installer_url}") - - logging.info("Running MSYS2 installer...") - logging.debug(f"Installing MSYS2 at {msys2_root_path}") - run_command(f"{installer_name} -y -oC:\\") - - logging.info("Updating MSYS2 packages...") - run_command(f"{msys2_root_path}\\usr\\bin\\bash.exe -lc 'pacman -Syu --noconfirm'") - - logging.info("Editing Environment Variables...") - logging.debug( - f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to PATH" - ) - # Set it permanently for the current user - commands = f""" - $oldPath = [Environment]::GetEnvironmentVariable("Path", "User") - $newPath = "{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" + $oldPath - [Environment]::SetEnvironmentVariable("Path", $newPath, "User") - """ - # Run the PowerShell commands - subprocess.check_call(["powershell", "-Command", commands]) - - # Add MSYS2 paths to the PATH environment variable for the current session - logging.debug( - f"Addiing {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to current enviornment." - ) - current_path = os.environ.get("PATH", "") - new_path = ( - f"{msys2_root_path}\\usr\\bin;{msys2_root_path}\\mingw64\\bin;{msys2_root_path}\\mingw32\\bin;" - + current_path - ) - os.environ["PATH"] = new_path - - logging.info("MSYS2 installed and updated successfully.") - logging.info("NOTE: Please restart your terminal before running this script again.") - - except subprocess.CalledProcessError as e: - logging.error(f"Error installing MSYS2: {e}", exc_info=True) - sys.exit(1) - finally: - if Path.exists(installer_name): - os.remove(installer_name) - - -def check_MediaProcessor(system): - if system == "Windows": - MediaProcessor_path = Path("MediaProcessor") / "build" / "MediaProcessor.exe" - else: - MediaProcessor_path = Path("MediaProcessor") / "build" / "MediaProcessor" - return MediaProcessor_path.exists() - - -def generate_virtualenv(): - logging.info(f"Generating virtual environment at: {VENV_DIR}") - run_command(f"{sys.executable} -m venv {str(VENV_DIR)} --system-site-packages") - logging.info("Successfully generated Virtual environment.") - - -def install_python_dependencies(system): - try: - if not VENV_DIR.exists(): - generate_virtualenv() - - logging.info("Installing Python dependencies...") - - if system == "Windows": - # Check for both `Scripts` and `bin` folders. - pip_scripts_path = VENV_DIR / "Scripts" / "pip.exe" - pip_bin_path = VENV_DIR / "bin" / "pip.exe" - - if not pip_scripts_path.exists() and not pip_bin_path.exists(): - logging.error(f"pip not found. Searched paths: [{pip_scripts_path}] and [{pip_bin_path}]") - sys.exit(1) - - if pip_bin_path.exists(): - pip_path = pip_bin_path - else: - pip_path = pip_scripts_path - - else: - pip_path = VENV_DIR / "bin" / "pip" - if not pip_path.exists(): - logging.error(f"pip not found. Searched paths: [{pip_path}]") - sys.exit(1) - - run_command(f"{str(pip_path)} install -r requirements.txt") - logging.info("Python dependencies installed.") - except Exception as e: - logging.error(f"Failed to install Python dependencies: {e}", exc_info=True) - sys.exit(1) +def ensure_MediaProcessor_build(system, re_build=False): + """Ensure that the MediaProcessor is build. If rebuild is true, it will rebuild the MediaProcessor.""" + if not MEDIAPROCESSOR_PATH.exists(): + os.makedirs(MEDIAPROCESSOR_PATH) - -def build_cpp_dependencies(): + MediaProcessor_binary_path = MEDIAPROCESSOR_PATH / ( + "MediaProcessor.exe" if system == "Windows" else "MediaProcessor" + ) + if MediaProcessor_binary_path.exists() and not re_build: + logging.debug(f"{str(MediaProcessor_binary_path)} exists. Skipping re-build.") + return try: - logging.info("Building MediaProcessor...") - build_dir = Path.join("MediaProcessor", "build") - if not Path.exists(build_dir): - os.makedirs(build_dir) - - run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=build_dir) - run_command("cmake --build . --config Release", cwd=build_dir) - + logging.info("building MediaProcessor.") + run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=MEDIAPROCESSOR_PATH) + run_command("cmake --build . --config Release", cwd=MEDIAPROCESSOR_PATH) logging.info("MediaProcessor built successfully.") except Exception as e: logging.error(f"Failed to build MediaProcessor: {e}", exc_info=True) @@ -380,6 +420,7 @@ def update_config(system): config = json.load(config_file) if system == "Windows": + logging.info("Updating config file.") for key, value in config.items(): if type(value) == str: config[key] = value.replace("/", "\\") @@ -404,13 +445,7 @@ def log_stream(stream, log_function): def launch_web_application(system): try: - if not VENV_DIR.exists(): - generate_virtualenv() - - if system == "Windows": - python_path = str(VENV_DIR / "Scripts" / "python.exe") - else: - python_path = str(VENV_DIR / "bin" / "python") + python_path = get_virutalenv_folder() / ("python.exe" if system == "Windows" else "python") # Start the backend app_process = subprocess.Popen( @@ -455,24 +490,14 @@ def main(): parser.add_argument("--debug-file", action="store_true", help="Set the debug file.") args = parser.parse_args() - """ - Initialise logging - check dependency - """ - system = platform.system() setup_logging(args.debug_level, args.debug_file) logging.info("Starting setup...") - for dependency in dependencies: - dependency.ensure_dependency_installed(system, args.install_dependencies) - - if args.install_dependencies: - install_python_dependencies(system) - - if args.rebuild or not check_MediaProcessor(system): - build_cpp_dependencies() + for dependency in required_dependencies: + dependency.ensure_installed(system) + ensure_MediaProcessor_build(system, args.rebuild) update_config(system) if args.app == "web": From 5aae3eb39c9651dd3828c57b52acb2652f4dbcc4 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 18:02:51 +0530 Subject: [PATCH 05/21] Add runtime config file. Now `config.json` is converted to `runtime_config.json` file for cross paltform compatibility. --- .gitignore | 3 +++ MediaProcessor/src/Engine.cpp | 2 +- MediaProcessor/src/main.cpp | 2 +- app.py | 9 +++++++-- config.json | 2 +- launcher.py | 10 +++++++--- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 3da1aec..89ee0c0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ __pycache__/ # C extensions *.so +# runtime config +runtime_config.json + # Distribution / packaging .Python build/ diff --git a/MediaProcessor/src/Engine.cpp b/MediaProcessor/src/Engine.cpp index 68ed419..cfa5157 100644 --- a/MediaProcessor/src/Engine.cpp +++ b/MediaProcessor/src/Engine.cpp @@ -14,7 +14,7 @@ Engine::Engine(const std::filesystem::path& mediaPath) bool Engine::processMedia() { ConfigManager& configManager = ConfigManager::getInstance(); - if (!configManager.loadConfig("config.json")) { + if (!configManager.loadConfig("runtime_config.json")) { std::cerr << "Error: Could not load configuration." << std::endl; return false; } diff --git a/MediaProcessor/src/main.cpp b/MediaProcessor/src/main.cpp index a55c331..1dce5cc 100644 --- a/MediaProcessor/src/main.cpp +++ b/MediaProcessor/src/main.cpp @@ -13,7 +13,7 @@ int main(int argc, char* argv[]) { * It supports both audio and video files, adapting the workflow based on the file type. * * Workflow: - * 1. Load configuration from "config.json". + * 1. Load configuration from "runtime_config.json". * 2. Determine if the input file is audio or video. * 3. For audio files: * - Directly process audio to isolate vocals. diff --git a/app.py b/app.py index 6345014..601c03f 100644 --- a/app.py +++ b/app.py @@ -3,6 +3,7 @@ import os import re import subprocess +from sys import exit from urllib.parse import urlparse import yt_dlp @@ -27,8 +28,12 @@ app = Flask(__name__) # Load config and set paths -with open("config.json") as config_file: - config = json.load(config_file) +try: + with open("runtime_config.json") as config_file: + config = json.load(config_file) +except FileNotFoundError: + print("Please run launcher.py.") + exit(1) # Define base paths using absolute references BASE_DIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/config.json b/config.json index 4b41997..36ad7a7 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "deep_filter_tarball_path": "MediaProcessor/res/DeepFilterNet3_ll_onnx.tar.gz", "deep_filter_encoder_path": "MediaProcessor/res/DeepFilterNet3_ll_onnx/tmp/export/enc.onnx", "deep_filter_decoder_path": "MediaProcessor/res/DeepFilterNet3_ll_onnx/tmp/export/df_dec.onnx", - "ffmpeg_path": "/usr/bin/ffmpeg", + "ffmpeg_path": "ffmpeg", "downloads_path": "downloads", "uploads_path": "uploads", "use_thread_cap": false, diff --git a/launcher.py b/launcher.py index 16047b0..d55dab1 100644 --- a/launcher.py +++ b/launcher.py @@ -14,6 +14,7 @@ from pathlib import Path CONFIG_FILE = Path("config.json") +RUNTIME_CONFIG_FILE = Path("runtime_config.json") VENV_NAME = "virtual_env" # For downloading python packages VENV_DIR = Path.cwd() / VENV_NAME # Generate the path for the virtual environment MEDIAPROCESSOR_PATH = Path("MediaProcessor") / "build" @@ -414,7 +415,10 @@ def ensure_MediaProcessor_build(system, re_build=False): sys.exit(1) -def update_config(system): +def ensure_runtime_config(system): + """ + make a new runtime_config file with Platform independent settings. + """ try: with open(CONFIG_FILE, "r") as config_file: config = json.load(config_file) @@ -425,7 +429,7 @@ def update_config(system): if type(value) == str: config[key] = value.replace("/", "\\") - with open(CONFIG_FILE, "w") as config_file: + with open(RUNTIME_CONFIG_FILE, "w") as config_file: json.dump(config, config_file, indent=4) except FileNotFoundError: @@ -498,7 +502,7 @@ def main(): dependency.ensure_installed(system) ensure_MediaProcessor_build(system, args.rebuild) - update_config(system) + ensure_runtime_config(system) if args.app == "web": launch_web_application(system) From 5d4c2f82d22a888e010884ec37bdeda6cda65c59 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 19:05:16 +0530 Subject: [PATCH 06/21] Fix `launcher.py` and `AudioProcessor.cpp` for Windows 1. Fix launcher.py runtime errors 2. utf_16 size error on Windows --- MediaProcessor/src/AudioProcessor.cpp | 8 +++++--- launcher.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/MediaProcessor/src/AudioProcessor.cpp b/MediaProcessor/src/AudioProcessor.cpp index 496d6c6..cefdd88 100644 --- a/MediaProcessor/src/AudioProcessor.cpp +++ b/MediaProcessor/src/AudioProcessor.cpp @@ -167,7 +167,9 @@ bool AudioProcessor::invokeDeepFilterFFI(fs::path chunkPath, DFState* df_state, std::vector& inputBuffer, std::vector& outputBuffer) { SF_INFO sfInfoIn; - SNDFILE* inputFile = sf_open(chunkPath.c_str(), SFM_READ, &sfInfoIn); + // First Convert to string + // On windows c_str converts to utf_16 but sf_open requires utf_8 + SNDFILE* inputFile = sf_open(chunkPath.string().c_str(), SFM_READ, &sfInfoIn); if (!inputFile) { std::cerr << "Error: Could not open input WAV file: " << chunkPath << std::endl; return false; @@ -175,7 +177,7 @@ bool AudioProcessor::invokeDeepFilterFFI(fs::path chunkPath, DFState* df_state, // Prepare output file fs::path processedChunkPath = m_processedChunksPath / chunkPath.filename(); - SNDFILE* outputFile = sf_open(processedChunkPath.c_str(), SFM_WRITE, &sfInfoIn); + SNDFILE* outputFile = sf_open(processedChunkPath.string().c_str(), SFM_WRITE, &sfInfoIn); if (!outputFile) { std::cerr << "Error: Could not open output WAV file: " << processedChunkPath << std::endl; sf_close(inputFile); @@ -214,7 +216,7 @@ bool AudioProcessor::filterChunks() { results.emplace_back(pool.enqueue([&, i]() { // Per-thread DFState instance DFState* df_state = - df_create(deepFilterTarballPath.c_str(), m_filterAttenuationLimit, nullptr); + df_create(deepFilterTarballPath.string().c_str(), m_filterAttenuationLimit, nullptr); if (!df_state) { std::cerr << "Error: Failed to insantiate DFState in thread." << std::endl; return false; diff --git a/launcher.py b/launcher.py index d55dab1..44ba6c1 100644 --- a/launcher.py +++ b/launcher.py @@ -354,7 +354,7 @@ def _get_install_commands_darwin(self): return [f"brew install {self.package_name.get('Darwin', self.name)}"] def _get_install_commands_windows(self): - MSYS2.ensure_installed() + MSYS2.ensure_installed("Windows") return [f"pacman -S --needed --noconfirm {self.package_name.get('Windows', self.name)}"] @@ -387,8 +387,8 @@ def _get_install_commands_windows(self): "Python Dependencies", check_cmd={"all": [check_python_dependecies_installed]}, install_cmd={ - "Windows": [f"{str(get_virutalenv_folder()/ "pip.exe")} install -r requirements.txt"], - "all": [f"{str(get_virutalenv_folder()/ "pip")} install -r requirements.txt"], + "Windows": [f"{str(get_virutalenv_folder()/ 'pip.exe')} install -r requirements.txt"], + "all": [f"{str(get_virutalenv_folder()/ 'pip')} install -r requirements.txt"], }, ), ] From 8c2722a7abf3c3b692c92d2a71b5333d8cc42fc5 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 22:49:33 +0530 Subject: [PATCH 07/21] Fix macOS architecture detection and library path issues in test suite --- MediaProcessor/cmake/test.cmake | 26 +++++++++++++++++-- MediaProcessor/tests/AudioProcessorTester.cpp | 1 + MediaProcessor/tests/TestUtils.cpp | 4 +-- MediaProcessor/tests/TestUtils.h | 2 +- MediaProcessor/tests/VideoProcessorTester.cpp | 16 +++--------- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/MediaProcessor/cmake/test.cmake b/MediaProcessor/cmake/test.cmake index 1bd67b9..0310695 100644 --- a/MediaProcessor/cmake/test.cmake +++ b/MediaProcessor/cmake/test.cmake @@ -14,6 +14,7 @@ endif() # Setup test media directory set(TEST_MEDIA_DIR "${CMAKE_SOURCE_DIR}/tests/TestMedia" CACHE PATH "Path to test media files") +file(TO_CMAKE_PATH "${TEST_MEDIA_DIR}" TEST_MEDIA_DIR) FetchContent_Declare( fmt @@ -22,8 +23,29 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(fmt) -# Common libraries for all test targets -set(COMMON_LIBRARIES gtest_main ${CMAKE_SOURCE_DIR}/lib/libdf.so ${SNDFILE_LIBRARIES} fmt::fmt) +if(APPLE) + include(CheckCXXCompilerFlag) + + # This fixes arch detection on macOS + check_cxx_compiler_flag("-arch arm64" COMPILER_SUPPORTS_ARM64) + + if(COMPILER_SUPPORTS_ARM64 AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "arm64") + set(DF_LIBRARY ${CMAKE_SOURCE_DIR}/lib/libdf.dylib) + else() + message(FATAL_ERROR "Unsupported macOS architecture: ${CMAKE_HOST_SYSTEM_PROCESSOR}") + # set(DF_LIBRARY ${CMAKE_SOURCE_DIR}/lib/libdf.dylib) + endif() +elseif(WIN32) + set(DF_LIBRARY ${CMAKE_SOURCE_DIR}/lib/libdf.dll.a) # for linktime + set(DF_DLL_PATH ${CMAKE_SOURCE_DIR}/lib/df.dll) # for runtime + +elseif(UNIX) + set(DF_LIBRARY ${CMAKE_SOURCE_DIR}/lib/libdf.so) +else() + message(FATAL_ERROR "Unsupported platform") +endif() + +set(COMMON_LIBRARIES gtest_main ${DF_LIBRARY} ${SNDFILE_LIBRARIES} fmt::fmt) # Macro for adding a test executable macro(add_test_executable name) diff --git a/MediaProcessor/tests/AudioProcessorTester.cpp b/MediaProcessor/tests/AudioProcessorTester.cpp index 368c9a5..0a4e691 100644 --- a/MediaProcessor/tests/AudioProcessorTester.cpp +++ b/MediaProcessor/tests/AudioProcessorTester.cpp @@ -24,6 +24,7 @@ class AudioProcessorTester : public ::testing::Test { } void SetUp() override { + testMediaPath.make_preferred(); fs::path currentPath = fs::current_path(); testVideoPath = testMediaPath / "test_video.mkv"; diff --git a/MediaProcessor/tests/TestUtils.cpp b/MediaProcessor/tests/TestUtils.cpp index 3a609df..da544b6 100644 --- a/MediaProcessor/tests/TestUtils.cpp +++ b/MediaProcessor/tests/TestUtils.cpp @@ -80,12 +80,12 @@ bool CompareFiles::compareFilesByteByByte(const fs::path& filePath1, const fs::p bool CompareFiles::compareAudioFiles(const fs::path& filePath1, const fs::path& filePath2, double tolerance, size_t chunkSize) { SF_INFO sfInfo1, sfInfo2; - SNDFILE* sndFile1 = sf_open(filePath1.c_str(), SFM_READ, &sfInfo1); + SNDFILE* sndFile1 = sf_open(filePath1.string().c_str(), SFM_READ, &sfInfo1); if (!sndFile1) { throw std::runtime_error("Failed to open file 1: " + filePath1.string()); } - SNDFILE* sndFile2 = sf_open(filePath2.c_str(), SFM_READ, &sfInfo2); + SNDFILE* sndFile2 = sf_open(filePath2.string().c_str(), SFM_READ, &sfInfo2); if (!sndFile2) { sf_close(sndFile1); throw std::runtime_error("Failed to open file 2: " + filePath2.string()); diff --git a/MediaProcessor/tests/TestUtils.h b/MediaProcessor/tests/TestUtils.h index b86d846..75f08fe 100644 --- a/MediaProcessor/tests/TestUtils.h +++ b/MediaProcessor/tests/TestUtils.h @@ -83,7 +83,7 @@ class TestConfigFile { (m_rootPath / "res/DeepFilterNet3_ll_onnx/tmp/export/enc.onnx")}, {"deep_filter_decoder_path", (m_rootPath / "res/DeepFilterNet3_ll_onnx/tmp/export/df_dec.onnx")}, - {"ffmpeg_path", "/usr/bin/ffmpeg"}, + {"ffmpeg_path", "ffmpeg"}, {"downloads_path", "downloads"}, {"uploads_path", "uploads"}, {"use_thread_cap", false}, diff --git a/MediaProcessor/tests/VideoProcessorTester.cpp b/MediaProcessor/tests/VideoProcessorTester.cpp index dfbf722..01eaf2a 100644 --- a/MediaProcessor/tests/VideoProcessorTester.cpp +++ b/MediaProcessor/tests/VideoProcessorTester.cpp @@ -28,6 +28,7 @@ class VideoProcessorTester : public ::testing::Test { } void SetUp() override { + testMediaPath.make_preferred(); testVideoPath = testMediaPath / "test_video.mkv"; testAudioPath = testMediaPath / "test_audio_processed.wav"; @@ -36,18 +37,6 @@ class VideoProcessorTester : public ::testing::Test { testOutputDir = fs::current_path() / "test_output"; fs::create_directories(testOutputDir); - - nlohmann::json jsonObject = { - {"ffmpeg_path", "/usr/bin/ffmpeg"}, - {"deep_filter_path", "MediaProcessor/res/deep-filter-0.5.6-x86_64-unknown-linux-musl"}, - {"downloads_path", "downloads"}, - {"uploads_path", "uploads"}, - {"use_thread_cap", true}, - {"max_threads_if_capped", 4}}; - testConfigFile.generateConfigFile("testConfig.json", jsonObject); - - ASSERT_TRUE(configManager.loadConfig(testConfigFile.getFilePath())) - << "Failed to load test configuration file."; } void TearDown() override { @@ -62,6 +51,9 @@ TEST_F(VideoProcessorTester, MergeMedia_MergesAudioAndVideoCorrectly) { * is already being checked within the audio tester. * Eventually we need check for sensible metrics here. */ + ConfigManager& configManager = ConfigManager::getInstance(); + ASSERT_TRUE(configManager.loadConfig(testConfigFile.getFilePath())) + << "Unable to Load TestConfigFile"; fs::path testOutputVideoPath = testOutputDir / "test_output_video.mp4"; VideoProcessor videoProcessor(testVideoPath, testAudioPath, testOutputVideoPath); From 1bedd2110429dac696d073908ec340f2ea7cd4e5 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Dec 2024 23:16:15 +0530 Subject: [PATCH 08/21] MINOR: formatted `AudioProcessor.cpp` --- MediaProcessor/src/AudioProcessor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MediaProcessor/src/AudioProcessor.cpp b/MediaProcessor/src/AudioProcessor.cpp index cefdd88..77d47c5 100644 --- a/MediaProcessor/src/AudioProcessor.cpp +++ b/MediaProcessor/src/AudioProcessor.cpp @@ -215,8 +215,8 @@ bool AudioProcessor::filterChunks() { for (int i = 0; i < m_numChunks; ++i) { results.emplace_back(pool.enqueue([&, i]() { // Per-thread DFState instance - DFState* df_state = - df_create(deepFilterTarballPath.string().c_str(), m_filterAttenuationLimit, nullptr); + DFState* df_state = df_create(deepFilterTarballPath.string().c_str(), + m_filterAttenuationLimit, nullptr); if (!df_state) { std::cerr << "Error: Failed to insantiate DFState in thread." << std::endl; return false; From 0a01bf3b1159f87007f00726586db12a2727b787 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Mon, 23 Dec 2024 10:51:19 +0530 Subject: [PATCH 09/21] Refactor `launcher.py` 1. Move config file handling and logging config to `Config` 2. Move Virtual env handling, running command, check internet connectivity to `Utils` class. 3. Move running web application to `WebApplication` class. --- launcher.py | 436 ++++++++++++++++++++++++++++------------------------ 1 file changed, 233 insertions(+), 203 deletions(-) diff --git a/launcher.py b/launcher.py index 44ba6c1..e964d37 100644 --- a/launcher.py +++ b/launcher.py @@ -12,94 +12,149 @@ import webbrowser from importlib.metadata import PackageNotFoundError, version from pathlib import Path +from typing import Optional -CONFIG_FILE = Path("config.json") -RUNTIME_CONFIG_FILE = Path("runtime_config.json") -VENV_NAME = "virtual_env" # For downloading python packages -VENV_DIR = Path.cwd() / VENV_NAME # Generate the path for the virtual environment MEDIAPROCESSOR_PATH = Path("MediaProcessor") / "build" -DEFAULT_CONFIG = { - "version": 1, - "formatters": { - "detailed": {"format": "%(asctime)s [%(levelname)s] [%(funcName)s(), %(lineno)d]: %(message)s"}, - "simple": {"format": "[%(levelname)s] %(message)s"}, - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "simple", - "stream": "ext://sys.stdout", +LOG_LOCK = threading.Lock() + + +class Config: + """ + Manages Config files and logging Configuration. + """ + + CONFIG_FILE_PATH = Path("config.json") + RUNTIME_CONFIG_FILE_PATH = Path("runtime_config.json") + DEFAULT_LOGGING_CONFIG = { + "version": 1, + "formatters": { + "detailed": {"format": "%(asctime)s [%(levelname)s] [%(funcName)s(), %(lineno)d]: %(message)s"}, + "simple": {"format": "[%(levelname)s] %(message)s"}, }, - }, - "root": {"level": "ERROR", "handlers": ["console"]}, -} + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple", + "stream": "ext://sys.stdout", + }, + }, + "root": {"level": "ERROR", "handlers": ["console"]}, + } + def __init__(self, system: str, log_level: str, log_file: bool = False) -> None: + self.setup_runtime_config(system) + self.setup_logging(log_level, log_file) -def setup_logging(log_level, log_file=False): - DEFAULT_CONFIG["root"]["level"] = log_level - if log_file: - DEFAULT_CONFIG["handlers"]["file"] = { - "class": "logging.handlers.RotatingFileHandler", - "formatter": "detailed", - "filename": f"logfile_{time.strftime('%Y%m%d_%H%M%S')}.log", - "maxBytes": 1024 * 1024 * 5, # 5MB - "backupCount": 3, - } - DEFAULT_CONFIG["root"]["handlers"].append("file") + def setup_runtime_config(self, system: str): + try: + with open(Config.CONFIG_FILE_PATH, "r") as config_file: + config = json.load(config_file) - logging.config.dictConfig(DEFAULT_CONFIG) + if system == "Windows": + logging.info("Updating config file.") + for key, value in config.items(): + if type(value) == str: + config[key] = value.replace("/", "\\") + with open(Config.RUNTIME_CONFIG_FILE_PATH, "w") as config_file: + json.dump(config, config_file, indent=4) -def run_command(command: str, cwd=None): - """ - Executes a shell command. + except FileNotFoundError: + logging.error(f"{Config.CONFIG_FILE_PATH} not found. Please create a default config.json file.") + sys.exit(1) + except Exception as e: + logging.error(f"Failed to update config: {e}", exc_info=True) + sys.exit(1) - Args: - command (str): The command to run. - cwd (str, optional): The working directory. Defaults to None. + def setup_logging(self, log_level: str, log_file=False): + Config.DEFAULT_LOGGING_CONFIG["root"]["level"] = log_level + if log_file: + Config.DEFAULT_LOGGING_CONFIG["handlers"]["file"] = { + "class": "logging.handlers.RotatingFileHandler", + "formatter": "detailed", + "filename": f"logfile_{time.strftime('%Y%m%d_%H%M%S')}.log", + "maxBytes": 1024 * 1024 * 5, # 5MB + "backupCount": 3, + } + Config.DEFAULT_LOGGING_CONFIG["root"]["handlers"].append("file") - Raises: - subprocess.CalledProcessError: If any command fails. - """ - logging.debug(f"Executing command: {command}") - process = subprocess.Popen( - command.split(), - cwd=cwd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - stdout, stderr = process.communicate() # wait for process to terminate - if stdout: - logging.debug(f"Command output: {stdout}") - if stderr: - logging.error(f"Command error: {stderr}") + logging.config.dictConfig(Config.DEFAULT_LOGGING_CONFIG) - if process.returncode != 0: - raise subprocess.CalledProcessError(process.returncode, command) +class Utils: + VENV_NAME = "virtual_env" # For downloading python packages + VENV_DIR_PATH = Path.cwd() / VENV_NAME # Generate the path for the virtual environment -def check_internet_connectivity(system: str) -> bool: - """ - Check if the system has internet connectivity by pinging Google's public DNS server. - """ - try: - if system == "Windows": - cmd = "ping -n 1 8.8.8.8" - else: - cmd = "ping -c 1 8.8.8.8" - logging.debug("Checking internet connectivity... ") - run_command(cmd) - logging.debug("Internet connectivity OK") - return True + @staticmethod + def run_command(command: str, cwd: Optional[Path] = None): + """ + Executes a shell command. - except Exception as e: - logging.error("No internet connection detected.") - logging.debug(f"Error: {e}") - return False + Args: + command (str): The command to run. + cwd (str, optional): The working directory. Defaults to None. + + Raises: + subprocess.CalledProcessError: If any command fails. + """ + logging.debug(f"Executing command: {command}") + process = subprocess.Popen( + command.split(), + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + stdout, stderr = process.communicate() # wait for process to terminate + if stdout: + logging.debug(f"Command output: {stdout}") + if stderr: + logging.error(f"Command error: {stderr}") + + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command) + + @staticmethod + def check_internet_connectivity(system) -> bool: + try: + cmd = "ping -n 1 8.8.8.8" if system == "Windows" else "ping -c 1 8.8.8.8" + Utils.run_command(cmd) + logging.debug("Internet connectivity OK") + return True + except Exception as e: + logging.error("No internet connection detected.") + logging.debug(f"Error: {e}") + return False + + @staticmethod + def ensure_virtualenv_exists(): + if Utils.VENV_DIR_PATH.exists(): + logging.debug("Virtual environment already exists.") + return + logging.info(f"Generating virtual environment at: {Utils.VENV_DIR_PATH}") + Utils.run_command(f"{sys.executable} -m venv {str(Utils.VENV_DIR_PATH)} --system-site-packages") + logging.info("Successfully generated Virtual environment.") + + @staticmethod + def get_virtualenv_folder() -> Path: + """ + get the path of the virtual environment binaries folder + """ + Utils.ensure_virtualenv_exists() + for folder in ["bin", "Scripts"]: + path = Utils.VENV_DIR_PATH / folder + logging.debug(f"Searching VENV Path: {path}") + if path.exists(): + return path + + logging.error("Could not locate virtual environment folders.") + sys.exit(1) def install_msys2(): + """ + Installs MSYS2 and Update Path enviornment variable for windows platform + """ try: installer_url = ( "https://github.com/msys2/msys2-installer/releases/download/nightly-x86_64/msys2-base-x86_64-latest.sfx.exe" @@ -110,14 +165,14 @@ def install_msys2(): logging.info("Downloading MSYS2 installer...") logging.debug(f"Installer URL: {installer_url}") - run_command(f"curl -L -o {installer_name} {installer_url}") + Utils.run_command(f"curl -L -o {installer_name} {installer_url}") logging.info("Running MSYS2 installer...") logging.debug(f"Installing MSYS2 at {msys2_root_path}") - run_command(f"{installer_name} -y -oC:\\") + Utils.run_command(f"{installer_name} -y -oC:\\") logging.info("Updating MSYS2 packages...") - run_command(f"{msys2_root_path}\\usr\\bin\\bash.exe -lc 'pacman -Syu --noconfirm'") + Utils.run_command(f"{msys2_root_path}\\usr\\bin\\bash.exe -lc 'pacman -Syu --noconfirm'") logging.info("Editing Environment Variables...") logging.debug( @@ -157,6 +212,9 @@ def install_msys2(): def check_python_dependecies_installed(): """ check if the packages in the requirements.txt are installed + + Note: + Currenly Supports `==` and `>=` operators only in the requirements.txt. """ try: requirements = open("requirements.txt", "r").readlines() @@ -192,29 +250,6 @@ def check_python_dependecies_installed(): sys.exit(1) -def ensure_virtualenv_exists(): - if VENV_DIR.exists(): - logging.debug("Virtual environment already exists.") - return - logging.info(f"Generating virtual environment at: {VENV_DIR}") - run_command(f"{sys.executable} -m venv {str(VENV_DIR)} --system-site-packages") - logging.info("Successfully generated Virtual environment.") - - -def get_virutalenv_folder() -> Path: - """ - get the path of the virtual environment binaries folder - """ - ensure_virtualenv_exists() - for folder in ["bin", "Scripts"]: - path = VENV_DIR / folder - if path.exists(): - return path - - logging.error("Could not locate virtual environment folders.") - sys.exit(1) - - class DependencyHandler: """ Represents a dependency with methods for checking and installing it. @@ -235,7 +270,8 @@ class DependencyHandler: for macos: brew for windows: msys2 - Note: If a function is provided for check_cmd ensure it raise FileNotFoundError if the dependency is not installed. + Note: + If a function is provided for check_cmd ensure it raise FileNotFoundError if the dependency is not installed. """ def __init__(self, name, package_name=None, check_cmd=None, install_cmd=None): @@ -290,7 +326,7 @@ def check_installed(self, system) -> bool: try: for cmd in commands: if type(cmd) == str: - run_command(cmd) + Utils.run_command(cmd) else: cmd() return True @@ -308,13 +344,15 @@ def install_dependency(self, system: str): Args: system (str): The operating system type. """ + if not Utils.check_internet_connectivity(system): + logging.error("Please Connect to a Internet Connection.") + sys.exit(1) try: - check_internet_connectivity(system) logging.debug(f"Installing {self.name} on {system}") commands = self._get_install_commands(system) for cmd in commands: if type(cmd) == str: - run_command(cmd) + Utils.run_command(cmd) else: cmd() logging.debug(f"Successfully installed {self.name}") @@ -362,36 +400,13 @@ def _get_install_commands_windows(self): "MSYS2", check_cmd={"Windows": ["pacman --version"]}, install_cmd={"Windows": [install_msys2]} ) -# pkg-config should come before sndfie and nlohmann-json -required_dependencies = [ - DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), - DependencyHandler( - "g++", - {"Windows": "mingw-w64-x86_64-gcc", "Darwin": "gcc"}, - {"all": ["g++ --version"]}, - {"Windows": ["pacman -S --needed --noconfirm base-devel mingw-w64-x86_64-toolchain"]}, - ), - DependencyHandler("pkg-config"), - DependencyHandler("ffmpeg", {"Windows": "mingw-w64-x86_64-ffmpeg"}, {"all": ["ffmpeg -version"]}), - DependencyHandler( - "libsndfile", - {"Windows": "mingw-w64-x86_64-libsndfile", "Linux": "libsndfile1-dev"}, - {"all": ["pkg-config --exists sndfile"]}, - ), - DependencyHandler( - "nlohmann-json", - {"Windows": "mingw-w64-x86_64-nlohmann-json", "Linux": "nlohmann-json3-dev"}, - {"all": ["pkg-config --exists nlohmann_json"]}, - ), - DependencyHandler( - "Python Dependencies", - check_cmd={"all": [check_python_dependecies_installed]}, - install_cmd={ - "Windows": [f"{str(get_virutalenv_folder()/ 'pip.exe')} install -r requirements.txt"], - "all": [f"{str(get_virutalenv_folder()/ 'pip')} install -r requirements.txt"], - }, - ), -] + +def log_stream(stream, log_function): + """Logs output from a stream.""" + for line in iter(stream.readline, ""): + with LOG_LOCK: + log_function(line.strip()) + stream.close() def ensure_MediaProcessor_build(system, re_build=False): @@ -407,107 +422,122 @@ def ensure_MediaProcessor_build(system, re_build=False): return try: logging.info("building MediaProcessor.") - run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=MEDIAPROCESSOR_PATH) - run_command("cmake --build . --config Release", cwd=MEDIAPROCESSOR_PATH) + Utils.run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=MEDIAPROCESSOR_PATH) + Utils.run_command("cmake --build . --config Release", cwd=MEDIAPROCESSOR_PATH) logging.info("MediaProcessor built successfully.") except Exception as e: logging.error(f"Failed to build MediaProcessor: {e}", exc_info=True) sys.exit(1) -def ensure_runtime_config(system): - """ - make a new runtime_config file with Platform independent settings. - """ - try: - with open(CONFIG_FILE, "r") as config_file: - config = json.load(config_file) - - if system == "Windows": - logging.info("Updating config file.") - for key, value in config.items(): - if type(value) == str: - config[key] = value.replace("/", "\\") - - with open(RUNTIME_CONFIG_FILE, "w") as config_file: - json.dump(config, config_file, indent=4) - - except FileNotFoundError: - logging.error(f"{CONFIG_FILE} not found. Please create a default config.json file.") - sys.exit(1) - except Exception as e: - logging.error(f"Failed to update config: {e}", exc_info=True) - sys.exit(1) +class WebApplication: + def __init__(self, system: str, log_level: str, log_file: bool = False): + self.required_dependencies = [ + DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), + DependencyHandler( + "g++", + {"Windows": "mingw-w64-x86_64-gcc", "Darwin": "gcc"}, + {"all": ["g++ --version"]}, + {"Windows": ["pacman -S --needed --noconfirm base-devel mingw-w64-x86_64-toolchain"]}, + ), + DependencyHandler("pkg-config"), + DependencyHandler("ffmpeg", {"Windows": "mingw-w64-x86_64-ffmpeg"}, {"all": ["ffmpeg -version"]}), + DependencyHandler( + "libsndfile", + {"Windows": "mingw-w64-x86_64-libsndfile", "Linux": "libsndfile1-dev"}, + {"all": ["pkg-config --exists sndfile"]}, + ), + DependencyHandler( + "nlohmann-json", + {"Windows": "mingw-w64-x86_64-nlohmann-json", "Linux": "nlohmann-json3-dev"}, + {"all": ["pkg-config --exists nlohmann_json"]}, + ), + DependencyHandler( + "Python Dependencies", + check_cmd={"all": [check_python_dependecies_installed]}, + install_cmd={ + "Windows": [f"{str(Utils.get_virtualenv_folder()/ 'pip.exe')} install -r requirements.txt"], + "all": [f"{str(Utils.get_virtualenv_folder()/ 'pip')} install -r requirements.txt"], + }, + ), + ] + self.system = system + self.URL = "http://127.0.0.1:8080" + self.tries = 5 + self.timeout = 0.5 + self.setup(log_level, log_file) + + def setup(self, log_level: str, log_file: bool): + """ + Installs the required dependencies and Setup Configuration for the web application. + """ + self.config = Config(self.system, log_level, log_file) + for dependency in self.required_dependencies: + dependency.ensure_installed(self.system) + def run(self): + try: + python_path = Utils.get_virtualenv_folder() / ("python.exe" if self.system == "Windows" else "python") -def log_stream(stream, log_function): - """Logs output from a stream.""" - for line in iter(stream.readline, ""): - log_function(line.strip()) - stream.close() + # Start the backend + app_process = subprocess.Popen( + [python_path, "app.py"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + atexit.register(app_process.terminate) + # Threads to handle stdout and stderr asynchronously + threading.Thread(target=log_stream, args=(app_process.stdout, logging.debug), daemon=True).start() + threading.Thread(target=log_stream, args=(app_process.stderr, logging.debug), daemon=True).start() -def launch_web_application(system): - try: - python_path = get_virutalenv_folder() / ("python.exe" if system == "Windows" else "python") + # Give the process some time to initialize + time.sleep(0.5) - # Start the backend - app_process = subprocess.Popen( - [python_path, "app.py"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - atexit.register(app_process.terminate) + # Check if the process is still running + if app_process.poll() is not None: + error_output = app_process.stderr.read() + logging.error(f"Error starting the backend: {error_output}") + sys.exit(1) - # Threads to handle stdout and stderr asynchronously - threading.Thread(target=log_stream, args=(app_process.stdout, logging.debug), daemon=True).start() - threading.Thread(target=log_stream, args=(app_process.stderr, logging.debug), daemon=True).start() + webbrowser.open("http://127.0.0.1:8080") - # Give the process some time to initialize - time.sleep(0.5) + logging.info("Web application running. Press Enter to stop.") + input() # Block until the user presses Enter - # Check if the process is still running - if app_process.poll() is not None: - error_output = app_process.stderr.read() - logging.error(f"Error starting the backend: {error_output}") + except Exception as e: + logging.error(f"An error occurred: {e}", exc_info=True) sys.exit(1) - webbrowser.open("http://127.0.0.1:8080") - logging.info("Web application running. Press Enter to stop.") - input() # Block until the user presses Enter - - except Exception as e: - logging.error(f"An error occurred: {e}", exc_info=True) - sys.exit(1) +Applications = { + "web": WebApplication, +} def main(): - parser = argparse.ArgumentParser(description="Setup for MediaProcessor Application") - parser.add_argument("--app", choices=["web", "none"], default="web", help="Specify launch mode") - parser.add_argument("--install-dependencies", action="store_true", default=True, help="Install dependencies.") + parser = argparse.ArgumentParser(description="Setup for MediaProcessor Application.") + parser.add_argument("--app", choices=["web", "none"], default="web", help="Specify launch mode (default=web).") + parser.add_argument("--install-only", action="store_true", default=False, help="Install dependencies Only.") parser.add_argument("--rebuild", action="store_true", help="Rebuild MediaProcessor") parser.add_argument( - "--debug-level", choices=["DEBUG", "INFO", "ERROR"], default="INFO", help="Set the debug output level." + "--log-level", choices=["DEBUG", "INFO", "ERROR"], default="INFO", help="Set the logging level (default=INFO)." ) - parser.add_argument("--debug-file", action="store_true", help="Set the debug file.") + parser.add_argument("--log-file", action="store_true", help="Outputs log in a log file.") args = parser.parse_args() system = platform.system() - setup_logging(args.debug_level, args.debug_file) - logging.info("Starting setup...") - - for dependency in required_dependencies: - dependency.ensure_installed(system) + if args.app == "none": + print("Please specify how you would like to launch the application, like --app=web. Exiting.") + sys.exit(0) + app = Applications[args.app](system, args.log_level, args.log_file) ensure_MediaProcessor_build(system, args.rebuild) - ensure_runtime_config(system) + if args.install_only: + sys.exit(0) - if args.app == "web": - launch_web_application(system) - else: - logging.info("Please specify how you would like to launch the application, like --app=web.") + app.run() if __name__ == "__main__": From 9592859affebd02db86311b5d2058abc7a4964d2 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Mon, 23 Dec 2024 20:36:31 +0530 Subject: [PATCH 10/21] Refactor `WebApplication class` in `launcher.py`: remove unused attribute, update sleep time and URL usage. --- launcher.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/launcher.py b/launcher.py index e964d37..b22396d 100644 --- a/launcher.py +++ b/launcher.py @@ -463,7 +463,6 @@ def __init__(self, system: str, log_level: str, log_file: bool = False): ] self.system = system self.URL = "http://127.0.0.1:8080" - self.tries = 5 self.timeout = 0.5 self.setup(log_level, log_file) @@ -493,7 +492,7 @@ def run(self): threading.Thread(target=log_stream, args=(app_process.stderr, logging.debug), daemon=True).start() # Give the process some time to initialize - time.sleep(0.5) + time.sleep(self.timeout) # Check if the process is still running if app_process.poll() is not None: @@ -501,7 +500,7 @@ def run(self): logging.error(f"Error starting the backend: {error_output}") sys.exit(1) - webbrowser.open("http://127.0.0.1:8080") + webbrowser.open(self.URL) logging.info("Web application running. Press Enter to stop.") input() # Block until the user presses Enter From 4886dfdc48acdc0468ccfa95de419816d2cce794 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 24 Dec 2024 17:44:49 +0530 Subject: [PATCH 11/21] Refactor `launcher.py`: Rename to stick to convention and Minor changes. 1. Rename `Config` to `ConfigManager`, required_dependencies to dependencies, virtualenv to venv 2. Add seprate function for updating config for windows --- launcher.py | 64 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/launcher.py b/launcher.py index b22396d..c8dfa3f 100644 --- a/launcher.py +++ b/launcher.py @@ -18,7 +18,7 @@ LOG_LOCK = threading.Lock() -class Config: +class ConfigManager: """ Manages Config files and logging Configuration. """ @@ -45,40 +45,42 @@ def __init__(self, system: str, log_level: str, log_file: bool = False) -> None: self.setup_runtime_config(system) self.setup_logging(log_level, log_file) + def update_config_for_windows(self, configuration): + for key, value in configuration.items(): + if type(value) == str: + configuration[key] = value.replace("/", "\\") # change / to \ for windows path compatibility + def setup_runtime_config(self, system: str): try: - with open(Config.CONFIG_FILE_PATH, "r") as config_file: + with open(ConfigManager.CONFIG_FILE_PATH, "r") as config_file: config = json.load(config_file) if system == "Windows": - logging.info("Updating config file.") - for key, value in config.items(): - if type(value) == str: - config[key] = value.replace("/", "\\") + self.update_config_for_windows(config) - with open(Config.RUNTIME_CONFIG_FILE_PATH, "w") as config_file: + with open(ConfigManager.RUNTIME_CONFIG_FILE_PATH, "w") as config_file: json.dump(config, config_file, indent=4) except FileNotFoundError: - logging.error(f"{Config.CONFIG_FILE_PATH} not found. Please create a default config.json file.") + logging.error(f"{ConfigManager.CONFIG_FILE_PATH} not found. Please create a default config.json file.") sys.exit(1) except Exception as e: logging.error(f"Failed to update config: {e}", exc_info=True) sys.exit(1) def setup_logging(self, log_level: str, log_file=False): - Config.DEFAULT_LOGGING_CONFIG["root"]["level"] = log_level + ConfigManager.DEFAULT_LOGGING_CONFIG["root"]["level"] = log_level if log_file: - Config.DEFAULT_LOGGING_CONFIG["handlers"]["file"] = { + ConfigManager.DEFAULT_LOGGING_CONFIG["handlers"]["file"] = { "class": "logging.handlers.RotatingFileHandler", "formatter": "detailed", "filename": f"logfile_{time.strftime('%Y%m%d_%H%M%S')}.log", "maxBytes": 1024 * 1024 * 5, # 5MB "backupCount": 3, } - Config.DEFAULT_LOGGING_CONFIG["root"]["handlers"].append("file") + ConfigManager.DEFAULT_LOGGING_CONFIG["root"]["handlers"].append("file") - logging.config.dictConfig(Config.DEFAULT_LOGGING_CONFIG) + logging.config.dictConfig(ConfigManager.DEFAULT_LOGGING_CONFIG) class Utils: @@ -127,7 +129,7 @@ def check_internet_connectivity(system) -> bool: return False @staticmethod - def ensure_virtualenv_exists(): + def ensure_venv_exists(): if Utils.VENV_DIR_PATH.exists(): logging.debug("Virtual environment already exists.") return @@ -136,13 +138,13 @@ def ensure_virtualenv_exists(): logging.info("Successfully generated Virtual environment.") @staticmethod - def get_virtualenv_folder() -> Path: + def get_venv_binaries_directory() -> Path: """ - get the path of the virtual environment binaries folder + get the path of the virtual environment binaries directory """ - Utils.ensure_virtualenv_exists() - for folder in ["bin", "Scripts"]: - path = Utils.VENV_DIR_PATH / folder + Utils.ensure_venv_exists() + for directory_name in ["bin", "Scripts"]: + path = Utils.VENV_DIR_PATH / directory_name logging.debug(f"Searching VENV Path: {path}") if path.exists(): return path @@ -176,7 +178,7 @@ def install_msys2(): logging.info("Editing Environment Variables...") logging.debug( - f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to PATH" + f"Adding {msys2_root_path}\\usr\\bin, {msys2_root_path}\\mingw64\\bin, {msys2_root_path}\\mingw32\\bin to PATH for current user." ) # Set it permanently for the current user commands = f""" @@ -209,7 +211,7 @@ def install_msys2(): os.remove(installer_name) -def check_python_dependecies_installed(): +def validate_python_dependencies(): """ check if the packages in the requirements.txt are installed @@ -409,7 +411,7 @@ def log_stream(stream, log_function): stream.close() -def ensure_MediaProcessor_build(system, re_build=False): +def build_processing_engine(system, re_build=False): """Ensure that the MediaProcessor is build. If rebuild is true, it will rebuild the MediaProcessor.""" if not MEDIAPROCESSOR_PATH.exists(): os.makedirs(MEDIAPROCESSOR_PATH) @@ -432,7 +434,7 @@ def ensure_MediaProcessor_build(system, re_build=False): class WebApplication: def __init__(self, system: str, log_level: str, log_file: bool = False): - self.required_dependencies = [ + self.dependencies = [ DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), DependencyHandler( "g++", @@ -454,10 +456,10 @@ def __init__(self, system: str, log_level: str, log_file: bool = False): ), DependencyHandler( "Python Dependencies", - check_cmd={"all": [check_python_dependecies_installed]}, + check_cmd={"all": [validate_python_dependencies]}, install_cmd={ - "Windows": [f"{str(Utils.get_virtualenv_folder()/ 'pip.exe')} install -r requirements.txt"], - "all": [f"{str(Utils.get_virtualenv_folder()/ 'pip')} install -r requirements.txt"], + "Windows": [f"{str(Utils.get_venv_binaries_directory()/ 'pip.exe')} install -r requirements.txt"], + "all": [f"{str(Utils.get_venv_binaries_directory()/ 'pip')} install -r requirements.txt"], }, ), ] @@ -468,15 +470,15 @@ def __init__(self, system: str, log_level: str, log_file: bool = False): def setup(self, log_level: str, log_file: bool): """ - Installs the required dependencies and Setup Configuration for the web application. + Installs the dependencies and Setup Configuration for the web application. """ - self.config = Config(self.system, log_level, log_file) - for dependency in self.required_dependencies: + self.config = ConfigManager(self.system, log_level, log_file) + for dependency in self.dependencies: dependency.ensure_installed(self.system) def run(self): try: - python_path = Utils.get_virtualenv_folder() / ("python.exe" if self.system == "Windows" else "python") + python_path = Utils.get_venv_binaries_directory() / ("python.exe" if self.system == "Windows" else "python") # Start the backend app_process = subprocess.Popen( @@ -518,7 +520,7 @@ def run(self): def main(): parser = argparse.ArgumentParser(description="Setup for MediaProcessor Application.") parser.add_argument("--app", choices=["web", "none"], default="web", help="Specify launch mode (default=web).") - parser.add_argument("--install-only", action="store_true", default=False, help="Install dependencies Only.") + parser.add_argument("--install-only", action="store_true", default=False, help="Install dependencies only.") parser.add_argument("--rebuild", action="store_true", help="Rebuild MediaProcessor") parser.add_argument( "--log-level", choices=["DEBUG", "INFO", "ERROR"], default="INFO", help="Set the logging level (default=INFO)." @@ -532,7 +534,7 @@ def main(): sys.exit(0) app = Applications[args.app](system, args.log_level, args.log_file) - ensure_MediaProcessor_build(system, args.rebuild) + build_processing_engine(system, args.rebuild) if args.install_only: sys.exit(0) From 210ad4a8ba7360eb276d95b5cb19d591e92e1a2e Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 24 Dec 2024 17:51:42 +0530 Subject: [PATCH 12/21] Add logging statement in `launcher.py` to indicate MSYS2 PATH addition --- launcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/launcher.py b/launcher.py index c8dfa3f..125accd 100644 --- a/launcher.py +++ b/launcher.py @@ -188,6 +188,7 @@ def install_msys2(): """ # Run the PowerShell commands subprocess.check_call(["powershell", "-Command", commands]) + logging.info("Added MSYS2 to PATH environment variable Permanently for current user.") # Add MSYS2 paths to the PATH environment variable for the current session logging.debug( From 3cb403fd787da751d80dcb56c9304c150d20e8be Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 24 Dec 2024 18:15:28 +0530 Subject: [PATCH 13/21] Add port parameter to `WebApplication.run` method in `launcher.py`. --- launcher.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/launcher.py b/launcher.py index 125accd..68d8145 100644 --- a/launcher.py +++ b/launcher.py @@ -465,7 +465,8 @@ def __init__(self, system: str, log_level: str, log_file: bool = False): ), ] self.system = system - self.URL = "http://127.0.0.1:8080" + self.DEAFULT_URL = "http://127.0.0.1" + self.DEAFULT_PORT = 8080 self.timeout = 0.5 self.setup(log_level, log_file) @@ -477,7 +478,7 @@ def setup(self, log_level: str, log_file: bool): for dependency in self.dependencies: dependency.ensure_installed(self.system) - def run(self): + def run(self, port:Optional[int] = None): try: python_path = Utils.get_venv_binaries_directory() / ("python.exe" if self.system == "Windows" else "python") @@ -503,7 +504,9 @@ def run(self): logging.error(f"Error starting the backend: {error_output}") sys.exit(1) - webbrowser.open(self.URL) + url = f"{self.DEAFULT_URL}:{port if port else self.DEAFULT_PORT}" + logging.debug(f"Web application running on {url}.") + webbrowser.open(url) logging.info("Web application running. Press Enter to stop.") input() # Block until the user presses Enter From 2a982c682d666c02dfef59982308eddf41a9efca Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 24 Dec 2024 18:17:48 +0530 Subject: [PATCH 14/21] Rename `MEDIAPROCESSOR_PATH` to `PROCESSING_ENGINE_PATH` in `launcher.py` --- launcher.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/launcher.py b/launcher.py index 68d8145..8dd1d35 100644 --- a/launcher.py +++ b/launcher.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Optional -MEDIAPROCESSOR_PATH = Path("MediaProcessor") / "build" +PROCESSING_ENGINE_PATH = Path("MediaProcessor") / "build" LOG_LOCK = threading.Lock() @@ -414,10 +414,10 @@ def log_stream(stream, log_function): def build_processing_engine(system, re_build=False): """Ensure that the MediaProcessor is build. If rebuild is true, it will rebuild the MediaProcessor.""" - if not MEDIAPROCESSOR_PATH.exists(): - os.makedirs(MEDIAPROCESSOR_PATH) + if not PROCESSING_ENGINE_PATH.exists(): + os.makedirs(PROCESSING_ENGINE_PATH) - MediaProcessor_binary_path = MEDIAPROCESSOR_PATH / ( + MediaProcessor_binary_path = PROCESSING_ENGINE_PATH / ( "MediaProcessor.exe" if system == "Windows" else "MediaProcessor" ) if MediaProcessor_binary_path.exists() and not re_build: @@ -425,8 +425,8 @@ def build_processing_engine(system, re_build=False): return try: logging.info("building MediaProcessor.") - Utils.run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=MEDIAPROCESSOR_PATH) - Utils.run_command("cmake --build . --config Release", cwd=MEDIAPROCESSOR_PATH) + Utils.run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=PROCESSING_ENGINE_PATH) + Utils.run_command("cmake --build . --config Release", cwd=PROCESSING_ENGINE_PATH) logging.info("MediaProcessor built successfully.") except Exception as e: logging.error(f"Failed to build MediaProcessor: {e}", exc_info=True) @@ -478,7 +478,7 @@ def setup(self, log_level: str, log_file: bool): for dependency in self.dependencies: dependency.ensure_installed(self.system) - def run(self, port:Optional[int] = None): + def run(self, port: Optional[int] = None): try: python_path = Utils.get_venv_binaries_directory() / ("python.exe" if self.system == "Windows" else "python") From a25b39e00cad25f22f7236c2c8040c3b54de53bf Mon Sep 17 00:00:00 2001 From: Nikhil Date: Tue, 24 Dec 2024 19:35:26 +0530 Subject: [PATCH 15/21] Update installer URL from nightly to stable release for msys2 in `launcher.py` --- launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher.py b/launcher.py index 8dd1d35..fa4b218 100644 --- a/launcher.py +++ b/launcher.py @@ -159,7 +159,7 @@ def install_msys2(): """ try: installer_url = ( - "https://github.com/msys2/msys2-installer/releases/download/nightly-x86_64/msys2-base-x86_64-latest.sfx.exe" + "https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20241208.sfx.exe" ) installer_name = "msys2-installer.exe" From 81ed83c1ab68a128447be5117b9e168892ce0567 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 29 Dec 2024 13:26:00 +0530 Subject: [PATCH 16/21] Refactor launcher.py: Rename `get_venv_binaries_directory` to `get_venv_binaries_path`. --- launcher.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/launcher.py b/launcher.py index fa4b218..5234fc6 100644 --- a/launcher.py +++ b/launcher.py @@ -138,9 +138,9 @@ def ensure_venv_exists(): logging.info("Successfully generated Virtual environment.") @staticmethod - def get_venv_binaries_directory() -> Path: + def get_venv_binaries_path() -> Path: """ - get the path of the virtual environment binaries directory + get the path of the virtual environment binaries path """ Utils.ensure_venv_exists() for directory_name in ["bin", "Scripts"]: @@ -158,9 +158,7 @@ def install_msys2(): Installs MSYS2 and Update Path enviornment variable for windows platform """ try: - installer_url = ( - "https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20241208.sfx.exe" - ) + installer_url = "https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20241208.sfx.exe" installer_name = "msys2-installer.exe" msys2_root_path = "C:\\msys64" @@ -459,8 +457,8 @@ def __init__(self, system: str, log_level: str, log_file: bool = False): "Python Dependencies", check_cmd={"all": [validate_python_dependencies]}, install_cmd={ - "Windows": [f"{str(Utils.get_venv_binaries_directory()/ 'pip.exe')} install -r requirements.txt"], - "all": [f"{str(Utils.get_venv_binaries_directory()/ 'pip')} install -r requirements.txt"], + "Windows": [f"{str(Utils.get_venv_binaries_path()/ 'pip.exe')} install -r requirements.txt"], + "all": [f"{str(Utils.get_venv_binaries_path()/ 'pip')} install -r requirements.txt"], }, ), ] @@ -480,7 +478,7 @@ def setup(self, log_level: str, log_file: bool): def run(self, port: Optional[int] = None): try: - python_path = Utils.get_venv_binaries_directory() / ("python.exe" if self.system == "Windows" else "python") + python_path = Utils.get_venv_binaries_path() / ("python.exe" if self.system == "Windows" else "python") # Start the backend app_process = subprocess.Popen( From 6935987f0e2e557d59a9c1107fca0c953647c0ad Mon Sep 17 00:00:00 2001 From: Nikhil Date: Thu, 2 Jan 2025 11:03:20 +0530 Subject: [PATCH 17/21] Tooling: Refactor info messages and doxy comment in `launcher.py` --- launcher.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/launcher.py b/launcher.py index 5234fc6..8b34ecb 100644 --- a/launcher.py +++ b/launcher.py @@ -51,6 +51,7 @@ def update_config_for_windows(self, configuration): configuration[key] = value.replace("/", "\\") # change / to \ for windows path compatibility def setup_runtime_config(self, system: str): + logging.debug(f"Setting up runtime config for {system}") try: with open(ConfigManager.CONFIG_FILE_PATH, "r") as config_file: config = json.load(config_file) @@ -62,7 +63,9 @@ def setup_runtime_config(self, system: str): json.dump(config, config_file, indent=4) except FileNotFoundError: - logging.error(f"{ConfigManager.CONFIG_FILE_PATH} not found. Please create a default config.json file.") + logging.error( + f"{ConfigManager.CONFIG_FILE_PATH} not found. Please check a config file exists in project root." + ) sys.exit(1) except Exception as e: logging.error(f"Failed to update config: {e}", exc_info=True) @@ -131,7 +134,7 @@ def check_internet_connectivity(system) -> bool: @staticmethod def ensure_venv_exists(): if Utils.VENV_DIR_PATH.exists(): - logging.debug("Virtual environment already exists.") + logging.debug(f"Virtual environment already exists at {Utils.VENV_DIR_PATH}.") return logging.info(f"Generating virtual environment at: {Utils.VENV_DIR_PATH}") Utils.run_command(f"{sys.executable} -m venv {str(Utils.VENV_DIR_PATH)} --system-site-packages") @@ -214,6 +217,9 @@ def validate_python_dependencies(): """ check if the packages in the requirements.txt are installed + Raises: + FileNotFoundError : if the requirements are not installed. + Note: Currenly Supports `==` and `>=` operators only in the requirements.txt. """ @@ -232,7 +238,7 @@ def validate_python_dependencies(): required_version = tuple(map(int, required_version.strip().split("."))) installed_version = tuple(map(int, installed_version.strip().split("."))) - logging.debug(f"installed version of {package_name}: {installed_version}") + logging.debug(f"Installed version of {package_name}: {installed_version}") if (operator == ">=" and installed_version >= required_version) or ( operator == "==" and installed_version == required_version ): @@ -240,7 +246,7 @@ def validate_python_dependencies(): else: logging.debug( f"Version mismatch for {package_name}: " - f"installed {installed_version}, required {required_version}" + f"installed {installed_version}, required {required_version}, operator {operator}" ) raise FileNotFoundError except (PackageNotFoundError, FileNotFoundError): @@ -422,10 +428,10 @@ def build_processing_engine(system, re_build=False): logging.debug(f"{str(MediaProcessor_binary_path)} exists. Skipping re-build.") return try: - logging.info("building MediaProcessor.") + logging.info("building Processing Engine.") Utils.run_command("cmake -DCMAKE_BUILD_TYPE=Release ..", cwd=PROCESSING_ENGINE_PATH) Utils.run_command("cmake --build . --config Release", cwd=PROCESSING_ENGINE_PATH) - logging.info("MediaProcessor built successfully.") + logging.info("Processing Engine built successfully.") except Exception as e: logging.error(f"Failed to build MediaProcessor: {e}", exc_info=True) sys.exit(1) @@ -481,6 +487,7 @@ def run(self, port: Optional[int] = None): python_path = Utils.get_venv_binaries_path() / ("python.exe" if self.system == "Windows" else "python") # Start the backend + logging.debug("Starting backend") app_process = subprocess.Popen( [python_path, "app.py"], stdout=subprocess.PIPE, @@ -532,7 +539,7 @@ def main(): system = platform.system() if args.app == "none": - print("Please specify how you would like to launch the application, like --app=web. Exiting.") + print("Please specify a launch option, e.g. `--app=web`. Start with `--help` to see all options.") sys.exit(0) app = Applications[args.app](system, args.log_level, args.log_file) From dc7c8501863bb32d1e0790f8f6c2d9311c91bcd7 Mon Sep 17 00:00:00 2001 From: omeryusufyagci Date: Wed, 8 Jan 2025 22:33:29 +0100 Subject: [PATCH 18/21] Core: Remove local path ref from libdf.dylib install name This fixes an issue where the .dylib was pointing at a hardcoded local path instead of @rpath. --- MediaProcessor/lib/libdf.dylib | Bin 16248184 -> 16248184 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/MediaProcessor/lib/libdf.dylib b/MediaProcessor/lib/libdf.dylib index 2e421925c7cc1d8246250a0ebb912bfa423be0fc..33d68126da79050213ab1198c054e3baed7c9da9 100755 GIT binary patch delta 1083 zcmWmAd3Y8A0LSs(v_-7wbWF|Bq0&lXQf+9Vi$=;-Rx2CBbZqao?!^*nbF{8i!U`Rx zsF{voZJnmVtg*LTp)e}zzdrq*=kxsj|2EXrHEkfTbs(y^&BCxi&w~8?vQ};5CZ#1# zNsdlU8J9G%M^dJ5WShdnYhnwkf<-Yoq4?ZTe1OJWKoi0Urzsb55zV-mONgL3Ex44+ zXh|zt(}uRR<8rRxO0FW3t7%UMqPT`@xsL1UNGCeeg&XKfH@b5pJ&2|!y|{_q+)N*C z;Z|a}jlSGYKl(F(SmL;YJGqO2oNqFSySayZ8O#udGK}He$Nh{Tfd_bykvzmGM)NR_ z@FIL6A(c$YvVTnZZnE zF`GH$Fqap2kzD5S67zYPS9p~LEF_P7UgLG%;7t}$z+#rLlw~ZZkQJqVcw|R$m zDPj$4d5`z`fOV{A0~`5}Vm@LMoB5be*uqvy_>@vUV;kGq!A{EfoG>XJZZ*S@1_$LG{m7e{O?%qow}4F=Av J2?qW@aSn~VK5GB~ delta 1125 zcmWmAXIRVu0D$r5kYr>utdKpgvo{spiHJ}_;kx@@T`rxwE0>H2XV+OLk`b~s5X#8T zh>VcZQFhtmeDnJ7d*1J_t+c}4hGb_^aF$DwohZU$HWxTqQXh41&&IFh~q?T{zJR8bPiXBuYW6muS>sN|lE)j^-5Em3k&xr6#LX zgj%SLI)JFCPEO3S(uGEkRuEVC?P zQJ9BlXfYo;=wUz%VzB^5#9<-gu?UN?1WS>CWmt|xtiVdFf(fg!25Yen>#+eFk%VNJ zu?d^81zWKV7Hmffc3>w`u?uO~jXl_leMrZC9Kb Date: Sat, 18 Jan 2025 20:08:09 +0530 Subject: [PATCH 19/21] Tooling: Add logging for displaying log statements to launcher.py --- launcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/launcher.py b/launcher.py index 8b34ecb..5083048 100644 --- a/launcher.py +++ b/launcher.py @@ -537,6 +537,8 @@ def main(): parser.add_argument("--log-file", action="store_true", help="Outputs log in a log file.") args = parser.parse_args() + print('Setting up....') + system = platform.system() if args.app == "none": print("Please specify a launch option, e.g. `--app=web`. Start with `--help` to see all options.") From 0f9adc973fbc3653d71878e41d407b0330832c99 Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sat, 18 Jan 2025 20:10:39 +0530 Subject: [PATCH 20/21] Tooling: Minor code formatting --- launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher.py b/launcher.py index 5083048..b3ba32b 100644 --- a/launcher.py +++ b/launcher.py @@ -537,7 +537,7 @@ def main(): parser.add_argument("--log-file", action="store_true", help="Outputs log in a log file.") args = parser.parse_args() - print('Setting up....') + print("Setting up....") system = platform.system() if args.app == "none": From 23342d760991e0f403c57b32d74050cad924b2f9 Mon Sep 17 00:00:00 2001 From: omeryusufyagci Date: Sun, 19 Jan 2025 21:48:59 +0100 Subject: [PATCH 21/21] Tooling: Add logging to Web app and FIXMEs for launcher.py --- launcher.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/launcher.py b/launcher.py index b3ba32b..4e8f736 100644 --- a/launcher.py +++ b/launcher.py @@ -439,6 +439,14 @@ def build_processing_engine(system, re_build=False): class WebApplication: def __init__(self, system: str, log_level: str, log_file: bool = False): + + # FIXME: ConfigManager is not initialized yet so logging is not setup. + # Following log statements to be swapped once this is fixed. + # logging.info("Setting up Web Application. This will resolve all dependencies, and may take a few minutes.") + # logging.info("Run with `--log-level DEBUG` for more detailed output.") + print("Setting up Web Application. This will resolve all dependencies, and may take a few minutes.") + print("Run with `--log-level DEBUG` for more detailed output.") + self.dependencies = [ DependencyHandler("cmake", {"Windows": "mingw-w64-x86_64-cmake"}, {"all": ["cmake --version"]}), DependencyHandler( @@ -478,6 +486,11 @@ def setup(self, log_level: str, log_file: bool): """ Installs the dependencies and Setup Configuration for the web application. """ + + """ FIXME: Common dependencies shouldn't be resolved here. + This should be exclusive to the WebApplication class dependencies. + """ + self.config = ConfigManager(self.system, log_level, log_file) for dependency in self.dependencies: dependency.ensure_installed(self.system) @@ -537,13 +550,19 @@ def main(): parser.add_argument("--log-file", action="store_true", help="Outputs log in a log file.") args = parser.parse_args() - print("Setting up....") + """ FIXME: Control flow should be more transparent. + It's unclear where dependencies are resolved (currently delegated to WebApplication). + Similar to building the core, resolving common dependencies should be explicit. + Consider adding init/cleanup functions to fix these without cluttering main. + """ system = platform.system() if args.app == "none": print("Please specify a launch option, e.g. `--app=web`. Start with `--help` to see all options.") sys.exit(0) + # FIXME: App starts before ConfigManager is initialized leading to limited intermediary logging capability. + # To be handled separately likely in the to-be-added init function. app = Applications[args.app](system, args.log_level, args.log_file) build_processing_engine(system, args.rebuild) if args.install_only: