From 340d0cb76c17cfa6ca994c3e37ab1cb1225f7bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Boisselier?= Date: Sun, 29 Sep 2024 20:30:54 +0200 Subject: [PATCH] added automatic temporary swap file usage --- README.md | 6 +++ install.sh | 47 +++++++++++++++--- install_swap_access.sh | 93 ++++++++++++++++++++++++++++++++++ shaketune/shaketune.py | 34 ++++++++++--- shaketune/shaketune_config.py | 2 + shaketune/swap_manager.py | 94 +++++++++++++++++++++++++++++++++++ 6 files changed, 261 insertions(+), 15 deletions(-) create mode 100755 install_swap_access.sh create mode 100644 shaketune/swap_manager.py diff --git a/README.md b/README.md index fb37b8f..61722e0 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,12 @@ Follow these steps to install Shake&Tune on your printer: # RAM, and should work for everyone. However, if you are using a powerful computer, you may # wish to increase this value to keep more measurements in memory (e.g., 15-20) before writing # the chunk and avoid stressing the SD card too much. + # temporary_swap_size: 0 + # This allows to specify the size in MB of an additional temporary swap file that will be dynamically + # created on the system to avoid running out of memory. This should help mitigating Klipper Timer + # Too Close errors that can occur on low-end devices with little RAM like the CB1 when processing + # large measurements. If you want to use this setting, be sure to have enough disk space available + # in your home folder, and a value like 512 or 1024 (MB) should be enough in most cases. # dpi: 300 # Controls the resolution of the generated graphs. The default value of 300 dpi was optimized # and strikes a balance between performance and readability, ensuring that graphs are clear diff --git a/install.sh b/install.sh index 7d59e13..e66ea19 100755 --- a/install.sh +++ b/install.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script is used to install the Shake&Tune module on a Klipper machine + + USER_CONFIG_PATH="${HOME}/printer_data/config" MOONRAKER_CONFIG="${HOME}/printer_data/config/moonraker.conf" KLIPPER_PATH="${HOME}/klipper" @@ -11,7 +14,6 @@ K_SHAKETUNE_PATH="${HOME}/klippain_shaketune" set -eu export LC_ALL=C - function preflight_checks { if [ "$EUID" -eq 0 ]; then echo "[PRE-CHECK] This script must not be run as root!" @@ -23,7 +25,7 @@ function preflight_checks { exit -1 fi - if [ "$(sudo systemctl list-units --full -all -t service --no-legend | grep -F 'klipper.service')" ]; then + if sudo systemctl is-active --quiet klipper; then printf "[PRE-CHECK] Klipper service found! Continuing...\n\n" else echo "[ERROR] Klipper service not found, please install Klipper first!" @@ -40,7 +42,7 @@ function is_package_installed { } function install_package_requirements { - packages=("libopenblas-dev" "libatlas-base-dev") + packages=("libopenblas-dev" "libatlas-base-dev" "sudo") packages_to_install="" for package in "${packages[@]}"; do @@ -76,6 +78,13 @@ function check_download { fi } +function setup_shaketune_sudo { + echo "[SETUP] Setting up sudo permissions for Shake&Tune swap file management..." + chmod +x ${K_SHAKETUNE_PATH}/install_swap_access.sh + ${K_SHAKETUNE_PATH}/install_swap_access.sh + printf "\n" +} + function setup_venv { if [ ! -d "${KLIPPER_VENV_PATH}" ]; then echo "[ERROR] Klipper's Python virtual environment not found!" @@ -83,7 +92,7 @@ function setup_venv { fi if [ -d "${OLD_K_SHAKETUNE_VENV}" ]; then - echo "[INFO] Old K-Shake&Tune virtual environement found, cleaning it!" + echo "[INFO] Old K-Shake&Tune virtual environment found, cleaning it!" rm -rf "${OLD_K_SHAKETUNE_VENV}" fi @@ -101,12 +110,12 @@ function link_extension { if [ -d "${HOME}/klippain_config" ] && [ -f "${USER_CONFIG_PATH}/.VERSION" ]; then if [ -d "${USER_CONFIG_PATH}/scripts/K-ShakeTune" ]; then echo "[INFO] Old K-Shake&Tune macro folder found, cleaning it!" - rm -d "${USER_CONFIG_PATH}/scripts/K-ShakeTune" + rm -rf "${USER_CONFIG_PATH}/scripts/K-ShakeTune" fi else if [ -d "${USER_CONFIG_PATH}/K-ShakeTune" ]; then echo "[INFO] Old K-Shake&Tune macro folder found, cleaning it!" - rm -d "${USER_CONFIG_PATH}/K-ShakeTune" + rm -rf "${USER_CONFIG_PATH}/K-ShakeTune" fi fi } @@ -147,9 +156,31 @@ printf "=============================================\n\n" # Run steps preflight_checks check_download +setup_shaketune_sudo setup_venv link_extension link_module add_updater -restart_klipper -restart_moonraker + + +echo "[POST-INSTALL] Shake&Tune installation complete!" + +# Ask the user if he want to reboot now +read -p "Do you want to reboot now? [y/N] " answer1 +case $answer1 in + [yY][eE][sS]|[yY]) + sudo reboot + ;; + *) + # Ask the user if he still want to restart Klipper and Moonraker + read -p "Ok, but do you want to at least restart Klipper and Moonraker? [y/N] " answer2 + case $answer2 in + [yY][eE][sS]|[yY]) + restart_klipper + restart_moonraker + ;; + *) + echo "Ok, but just a heads-up: Shake&Tune won't be available until you restart your printer!" + esac + ;; +esac diff --git a/install_swap_access.sh b/install_swap_access.sh new file mode 100755 index 0000000..4b169ae --- /dev/null +++ b/install_swap_access.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Sets up sudo permissions for the shaketune module to allow +# the user to create and delete a swap file without requiring a password + + +set -e + +SUDOERS_DIR='/etc/sudoers.d' +SUDOERS_FILE='020-sudo-for-shaketune' +NEW_GROUP='shaketunesudo' + + +verify_ready() { + if [ "$EUID" -eq 0 ]; then + echo "This script must not run as root" + exit -1 + fi +} + +create_sudoers_file() { + SCRIPT_TEMP_PATH=/tmp + + echo "Creating ${SUDOERS_FILE} ..." + sudo rm -f $SCRIPT_TEMP_PATH/$SUDOERS_FILE + sudo tee $SCRIPT_TEMP_PATH/$SUDOERS_FILE > /dev/null << EOF +Cmnd_Alias SWAP_CREATE = /usr/bin/fallocate -l * /home/*/shaketune_swap, /bin/dd if=/dev/zero of=/home/*/shaketune_swap bs=* count=* +Cmnd_Alias SWAP_SETUP = /sbin/mkswap /home/*/shaketune_swap, /sbin/swapon /home/*/shaketune_swap +Cmnd_Alias SWAP_REMOVE = /sbin/swapoff /home/*/shaketune_swap, /bin/rm /home/*/shaketune_swap + +%${NEW_GROUP} ALL=(root) NOPASSWD: SWAP_CREATE, SWAP_SETUP, SWAP_REMOVE +EOF +} + +verify_syntax() { + if command -v visudo &> /dev/null; then + echo "Verifying syntax of ${SUDOERS_FILE}..." + if sudo visudo -cf $SCRIPT_TEMP_PATH/$SUDOERS_FILE; then + VERIFY_STATUS=0 + echo "Syntax OK" + else + echo "Syntax Error: Check file at $SCRIPT_TEMP_PATH/$SUDOERS_FILE" + exit 1 + fi + else + VERIFY_STATUS=0 + echo "Command 'visudo' not found. Skipping syntax verification." + fi +} + +install_sudoers_file() { + verify_syntax + if [ $VERIFY_STATUS -eq 0 ]; then + echo "Installing sudoers file..." + sudo chmod 0440 $SCRIPT_TEMP_PATH/$SUDOERS_FILE + sudo cp $SCRIPT_TEMP_PATH/$SUDOERS_FILE $SUDOERS_DIR/$SUDOERS_FILE + else + exit 1 + fi +} + +add_new_group() { + if ! getent group $NEW_GROUP &> /dev/null; then + echo "Creating group ${NEW_GROUP}..." + sudo groupadd --system $NEW_GROUP + else + echo "Group ${NEW_GROUP} already exists." + fi +} + +add_user_to_group() { + if groups $USER | grep -qw $NEW_GROUP; then + echo "User ${USER} is already in group ${NEW_GROUP}." + else + echo "Adding user ${USER} to group ${NEW_GROUP}..." + sudo usermod -aG $NEW_GROUP $USER + fi +} + +clean_temp() { + sudo rm -f $SCRIPT_TEMP_PATH/$SUDOERS_FILE +} + + +# Run steps +verify_ready +create_sudoers_file +install_sudoers_file +add_new_group +add_user_to_group +clean_temp + +exit 0 diff --git a/shaketune/shaketune.py b/shaketune/shaketune.py index 1d05e72..b0fd5d9 100644 --- a/shaketune/shaketune.py +++ b/shaketune/shaketune.py @@ -29,14 +29,16 @@ from .helpers.console_output import ConsoleOutput from .shaketune_config import ShakeTuneConfig from .shaketune_process import ShakeTuneProcess +from .swap_manager import SwapManager DEFAULT_FOLDER = '~/printer_data/config/ShakeTune_results' -DEFAULT_NUMBER_OF_RESULTS = 10 -DEFAULT_KEEP_RAW_DATA = False -DEFAULT_DPI = 150 -DEFAULT_TIMEOUT = 600 -DEFAULT_SHOW_MACROS = True +DEFAULT_NUMBER_OF_RESULTS = 10 # Number of results to keep in the results folder before rotating them +DEFAULT_KEEP_RAW_DATA = False # Whether to also tore the .stdata files in the results folder +DEFAULT_DPI = 150 # Resolution of the generated graphs +DEFAULT_TIMEOUT = 600 # Maximum processing time (in seconds) to allow to Shake&Tune for generating graphs +DEFAULT_SHOW_MACROS = True # Whether to show the Shake&Tune macros in the web UI DEFAULT_MEASUREMENTS_CHUNK_SIZE = 2 # Maximum number of measurements to keep in memory at once +DEFAULT_TEMP_SWAP_SIZE_MB = 0 # 0 means no swap file is created ST_COMMANDS = { 'EXCITATE_AXIS_AT_FREQ': ( 'Maintain a specified excitation frequency for a period ' @@ -82,10 +84,18 @@ def _initialize_config(self, config) -> None: keep_raw_data = config.getboolean('keep_raw_data', default=DEFAULT_KEEP_RAW_DATA) dpi = config.getint('dpi', default=DEFAULT_DPI, minval=100, maxval=500) m_chunk_size = config.getint('measurements_chunk_size', default=DEFAULT_MEASUREMENTS_CHUNK_SIZE, minval=2) - self._st_config = ShakeTuneConfig(result_folder_path, keep_n_results, keep_raw_data, m_chunk_size, dpi) - + temp_swap_size_mb = config.getint('temporary_swap_size', default=DEFAULT_TEMP_SWAP_SIZE_MB, minval=0) + self._st_config = ShakeTuneConfig( + result_folder_path, + keep_n_results, + keep_raw_data, + m_chunk_size, + temp_swap_size_mb, + dpi, + ) self.timeout = config.getfloat('timeout', DEFAULT_TIMEOUT, above=0.0) self._show_macros = config.getboolean('show_macros_in_webui', default=DEFAULT_SHOW_MACROS) + self._swap_manager = SwapManager(temp_swap_size_mb) # Create the Klipper commands to allow the user to run Shake&Tune's tools def _register_commands(self) -> None: @@ -160,7 +170,9 @@ def cmd_EXCITATE_AXIS_AT_FREQ(self, gcmd) -> None: static_freq_graph_creator, self.timeout, ) + self._swap_manager.add_swap() excitate_axis_at_freq(gcmd, self._config, st_process) + self._swap_manager.remove_swap() def cmd_AXES_MAP_CALIBRATION(self, gcmd) -> None: ConsoleOutput.print(f'Shake&Tune version: {ShakeTuneConfig.get_git_version()}') @@ -171,7 +183,9 @@ def cmd_AXES_MAP_CALIBRATION(self, gcmd) -> None: axes_map_graph_creator, self.timeout, ) + self._swap_manager.add_swap() axes_map_calibration(gcmd, self._config, st_process) + self._swap_manager.remove_swap() def cmd_COMPARE_BELTS_RESPONSES(self, gcmd) -> None: ConsoleOutput.print(f'Shake&Tune version: {ShakeTuneConfig.get_git_version()}') @@ -182,7 +196,9 @@ def cmd_COMPARE_BELTS_RESPONSES(self, gcmd) -> None: belt_graph_creator, self.timeout, ) + self._swap_manager.add_swap() compare_belts_responses(gcmd, self._config, st_process) + self._swap_manager.remove_swap() def cmd_AXES_SHAPER_CALIBRATION(self, gcmd) -> None: ConsoleOutput.print(f'Shake&Tune version: {ShakeTuneConfig.get_git_version()}') @@ -193,7 +209,9 @@ def cmd_AXES_SHAPER_CALIBRATION(self, gcmd) -> None: shaper_graph_creator, self.timeout, ) + self._swap_manager.add_swap() axes_shaper_calibration(gcmd, self._config, st_process) + self._swap_manager.remove_swap() def cmd_CREATE_VIBRATIONS_PROFILE(self, gcmd) -> None: ConsoleOutput.print(f'Shake&Tune version: {ShakeTuneConfig.get_git_version()}') @@ -204,4 +222,6 @@ def cmd_CREATE_VIBRATIONS_PROFILE(self, gcmd) -> None: vibration_profile_creator, self.timeout, ) + self._swap_manager.add_swap() create_vibrations_profile(gcmd, self._config, st_process) + self._swap_manager.remove_swap() diff --git a/shaketune/shaketune_config.py b/shaketune/shaketune_config.py index c3d1885..91d2ddf 100644 --- a/shaketune/shaketune_config.py +++ b/shaketune/shaketune_config.py @@ -31,6 +31,7 @@ def __init__( keep_n_results: int = 10, keep_raw_data: bool = False, chunk_size: int = 2, + temp_swap_size_mb: int = 0, dpi: int = 150, ) -> None: self._result_folder = result_folder @@ -38,6 +39,7 @@ def __init__( self.keep_n_results = keep_n_results self.keep_raw_data = keep_raw_data self.chunk_size = chunk_size + self.temp_swap_size_mb = temp_swap_size_mb self.dpi = dpi self.klipper_folder = KLIPPER_FOLDER diff --git a/shaketune/swap_manager.py b/shaketune/swap_manager.py new file mode 100644 index 0000000..ec3b2ef --- /dev/null +++ b/shaketune/swap_manager.py @@ -0,0 +1,94 @@ +# Shake&Tune: 3D printer analysis tools +# +# Copyright (C) 2022 - 2024 Félix Boisselier (Frix_x on Discord) +# Licensed under the GNU General Public License v3.0 (GPL-3.0) +# +# File: swap_manager.py +# Description: Implements the SwapManager class for managing the creation and +# activation of a temporary swap file on the system to avoid running +# out of memory when processing large files (useful for low end devices like CB1) + +import shutil +import subprocess +from pathlib import Path + +from .helpers.console_output import ConsoleOutput + +SWAP_FILE_PATH = Path.home() / 'shaketune_swap' + + +class SwapManager: + def __init__(self, swap_size_mb: int = 0) -> None: + self._swap_size_mb = swap_size_mb + self._swap_file_path = SWAP_FILE_PATH + self._swap_activated = False + + def is_swap_activated(self) -> bool: + return self._swap_activated + + def add_swap(self) -> None: + if self._swap_size_mb <= 0: + return + + # Check if swap file already exists and delete it if it does + if self._swap_file_path.exists(): + ConsoleOutput.print(f'Warning: {self._swap_file_path} already exists. Replacing it...') + self.remove_swap() + + # Check available disk space to be sure there is enough space for the swap file + total, used, free = shutil.disk_usage(self._swap_file_path.parent) + free_mb = free // (1024 * 1024) + if free_mb < self._swap_size_mb: + ConsoleOutput.print( + f'Warning: not enough disk space available ({free_mb} MB) to create the temporary swap file ' + f'that you asked for ({self._swap_size_mb} MB). It will not be created for this run...' + ) + return + + # Create the swap file and activate it + try: + fallocate_result = subprocess.run( + [ + 'sudo', + 'fallocate', + '-l', + f'{self._swap_size_mb}M', + str(self._swap_file_path), + ], + check=True, + ) + if fallocate_result.returncode != 0: + ConsoleOutput.print( + 'Warning: failed to allocate the temporary swap file using fallocate. ' + 'Falling back to dd, this may take a while...' + ) + subprocess.run( + [ + 'sudo', + 'dd', + 'if=/dev/zero', + f'of={self._swap_file_path}', + 'bs=1M', + f'count={self._swap_size_mb}', + ], + check=True, + ) + subprocess.run(['sudo', 'chmod', '600', str(self._swap_file_path)], check=True) + subprocess.run(['sudo', 'mkswap', str(self._swap_file_path)], check=True) + subprocess.run(['sudo', 'swapon', str(self._swap_file_path)], check=True) + self._swap_activated = True + ConsoleOutput.print(f'Temporary swap file of {self._swap_size_mb} MB activated') + except subprocess.CalledProcessError as err: + self.remove_swap() + raise RuntimeError('Failed to create and activate the temporary swap file!') from err + + def remove_swap(self) -> None: + if not self._swap_file_path.exists(): + return + + try: + if self._swap_activated: + subprocess.run(['sudo', 'swapoff', str(self._swap_file_path)], check=True) + subprocess.run(['sudo', 'rm', str(self._swap_file_path)], check=True) + except subprocess.CalledProcessError as err: + raise RuntimeError('Failed to deactivate and delete the temporary swap file!') from err