From aacf9551c27e3f49a1344d6d3733ec7d19812f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 16 Jan 2025 13:45:39 +0100 Subject: [PATCH 01/46] ova workflow redesing wip --- .../ansible_playbooks/ova_generator.yaml | 99 +++++++++-- .github/workflows/builder_OVA.yaml | 167 ++++++------------ ova/workflow_assets/Vagrantfile | 92 ++++++++++ ova/workflow_assets/generate_base_box.sh | 110 ++++++++++++ ova/workflow_assets/setup.sh | 89 ++++++++++ 5 files changed, 432 insertions(+), 125 deletions(-) create mode 100644 ova/workflow_assets/Vagrantfile create mode 100755 ova/workflow_assets/generate_base_box.sh create mode 100755 ova/workflow_assets/setup.sh diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 4e61d3d..3371d1f 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -2,30 +2,105 @@ become: true tasks: + - name: Enable Amazon Linux Extras repository + amazon.aws.amazon_linux_extras: + name: python3.8 + state: present + + - name: Install python3.8 and pip + yum: + name: + - python3.8 + - python3-pip + state: present + + - name: Configure Python 3.8 as default + alternatives: + name: python3 + path: /usr/bin/python3.8 + + - name: Update all the packages + yum: + name: '*' + state: latest + + - name: Install required tools for VirtualBox + yum: + name: + - gcc + - make + - perl + - bzip2 + - kernel-devel + - kernel-headers + - dkms + state: present + + - name: Add the VirtualBox repository + yum_repository: + name: virtualbox + description: "Oracle VirtualBox Repository" + baseurl: https://download.virtualbox.org/virtualbox/rpm/el/7/x86_64 + gpgcheck: yes + gpgkey: https://www.virtualbox.org/download/oracle_vbox.asc + enabled: yes + + - name: Install VirtualBox + yum: + name: VirtualBox-7.0 + state: present + + - name: Rebuild the VirtualBox kernel modules + command: "/sbin/vboxconfig" + + - name: Add Vagrant repository + yum_repository: + name: vagrant + description: "HashiCorp Vagrant" + baseurl: https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo + gpgcheck: yes + gpgkey: https://rpm.releases.hashicorp.com/gpg + enabled: yes + + - name: Install Vagrant + yum: + name: vagrant + state: present + - name: Install git shell: sudo yum install -y git - - name: Make build directory + - name: Create directory for the base VM file: - path: "{{ ova_path }}" + path: "/tmp/ova_directory" state: directory + mode: '0755' - - name: Copy ova directory - copy: - src: "../../../ova" - dest: "{{ ova_path }}" - - - name: Download the Wazuh installation assistant repository + - name: Download the Wazuh virtual machines repository git: - repo: "{{ wia_repository }}" - version: "{{ wia_branch }}" - dest: '/tmp/{{ wia_scripts }}' - depth: 1 + repo: "{{ wvm_repository }}" + version: "{{ wvm_branch }}" + dest: "/tmp/wazuh-virtual-machines" register: clone_result retries: 6 delay: 10 until: clone_result is success + - name: Create base box + shell: "./generate_base_box.sh" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + + - name: Add the created box + shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_scripts/al2023.box" + + - name: Deploy the VM using Vagrant + shell: "vagrant up" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + + # Migrate everything below to the Python script + - name: Set custom hostname command: "hostnamectl set-hostname wazuh-server" diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 81ac679..4406ff7 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -12,10 +12,14 @@ on: description: 'Branch or tag of the wazuh-virtual-machines repository' required: true default: '4.11.0' - WAZUH_INSTALLATION_ASSISTANT_REFERENCE: + wazuh_installation_assistant_reference: description: 'Branch or tag of the wazuh-installation-assistant repository' required: true default: '4.11.0' + wazuh_automation_reference: + description: 'Branch or tag of the wazuh-automation repository' + required: true + default: '4.11.0' WAZUH_PACKAGE_REPOSITORY: type: choice description: 'Wazuh package repository from which to download the packages' @@ -57,23 +61,24 @@ on: required: false env: - OVA_AMI: "ami-0d4bd55523ee67aa4" - INSTANCE_TYPE: "t2.xlarge" - SECURITY_GROUP: "sg-005cff996b335d497" - SUBNET: "subnet-0b6aea31fb32cffad" - TEMPORAL_S3_BUCKET: "warehouse.wazuh.com" + # OVA_AMI: "ami-0d4bd55523ee67aa4" + INSTANCE_TYPE: "metal" + # SECURITY_GROUP: "sg-005cff996b335d497" + # SUBNET: "subnet-0b6aea31fb32cffad" + # TEMPORAL_S3_BUCKET: "warehouse.wazuh.com" S3_BUCKET: ${{ vars.AWS_S3_BUCKET }} S3_PATH: "development/wazuh/4.x/secondary/ova" - OVA_ENVIRONMENT: "vmware" - CONTAINER_FORMAT: "ova" - TEMPORAL_S3_PATH: "trash/vm" - OVA_USER: "wazuh-user" - OVA_USER_PASSWORD: "wazuh" + # OVA_ENVIRONMENT: "vmware" + # CONTAINER_FORMAT: "ova" + # TEMPORAL_S3_PATH: "trash/vm" + # OVA_USER: "wazuh-user" + # OVA_USER_PASSWORD: "wazuh" INVENTORY_PATH: "/tmp/allocatorvm_ova" AWS_REGION: "us-east-1" OVA_PATH: "/var/provision/wazuh-virtual-machines" WIA_DIR: "wazuh-installation-assistant" WIA_REPOSITORY: "https://github.com/wazuh/wazuh-installation-assistant" + WVM_REPOSITORY: "https://github.com/wazuh/wazuh-virtual-machines" ANSIBLE_CALLBACK: "yaml" permissions: @@ -84,6 +89,9 @@ jobs: build_and_run: runs-on: ubuntu-latest steps: + - name: View parameters + run: echo "${{ toJson(inputs) }}" + - name: Install Python and create virtual environment run: | sudo apt-get update @@ -122,9 +130,6 @@ jobs: FILENAME_SHA="${FILENAME}.ova.sha512" echo "FILENAME_SHA=$FILENAME_SHA" >> $GITHUB_ENV - - name: View parameters - run: echo "${{ toJson(inputs) }}" - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -133,76 +138,43 @@ jobs: aws-region: "${{ env.AWS_REGION }}" role-duration-seconds: 18000 # Set the duration of the role session to 5 hours - - name: Install and config OpenVPN - run: | - sudo apt update - sudo apt install -y openvpn openvpn-systemd-resolved - echo "${{ secrets.CI_VPN_GITHUB }}" > vpn.ovpn - sudo openvpn --config "vpn.ovpn" --daemon - - - name: Wait for a VPN connection - id: vpn_connected - timeout-minutes: 10 - run: | - while ! ping -c2 10.10.0.252; do - sudo kill -9 `pidof openvpn`; - sudo openvpn --config "vpn.ovpn" --daemon; - sleep 30; - done + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.wazuh_automation_reference }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation - - name: Create OVA VM - id: alloc_vm_ova + - name: Install and set allocator requirements run: | - instance=$(aws ec2 run-instances --image-id "${{ env.OVA_AMI }}" --count 1 --instance-type "${{ env.INSTANCE_TYPE }}" --key-name Ephemeral \ - --security-group-ids "${{ env.SECURITY_GROUP }}" --subnet-id "${{ env.SUBNET }}" \ - --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=gha_${{ github.run_id }}_ova_build},{Key=team,Value=devops}]') - INSTANCE_ID=$(echo $instance | jq -r '.Instances[0].InstanceId') - echo "INSTANCE_ID=${INSTANCE_ID}" >> $GITHUB_ENV + pip3 install -r wazuh-automation/deployability/deps/requirements.txt - - name: Wait for instance to be running + - name: Execute allocator module that will create the base instance + id: alloc_vm run: | - MAX_RETRIES=40 - NUM_RETRIES=0 - while true; do - STATUS=$(aws ec2 describe-instances --instance-ids "${{ env.INSTANCE_ID }}" | jq -r '.Reservations[0].Instances[0].State.Name') - if [ "${STATUS}" == "running" ]; then - break - fi - sleep 30 - NUM_RETRIES=$((NUM_RETRIES+1)) - if [ ${NUM_RETRIES} -eq ${MAX_RETRIES} ]; then - echo "Error creating OVA VM" - aws ec2 terminate-instances --instance-ids "${{ env.INSTANCE_ID }}" - exit 1 - fi - done - ansible_host=$(aws ec2 describe-instances --instance-ids "${{ env.INSTANCE_ID }}" | jq -r '.Reservations[0].Instances[0].PrivateIpAddress') - mkdir -p ${{ env.INVENTORY_PATH }} - echo "[gha_instance]" > ${{ env.INVENTORY_PATH }}/inventory - echo "$ansible_host ansible_user=${{ env.OVA_USER }} ansible_password=${{ env.OVA_USER_PASSWORD }} ansible_ssh_common_args='-o StrictHostKeyChecking=no'" >> ${{ env.INVENTORY_PATH }}/inventory - echo "ANSIBLE_HOST=$ansible_host" >> $GITHUB_ENV + python3 wazuh-automation/deployability/modules/allocation/main.py --action create --provider aws --size ${{ env.INSTANCE_TYPE }} --composite-name ${{ env.COMPOSITE_NAME }} --working-dir ${{ env.ALLOCATOR_PATH }} \ + --track-output ${{ env.ALLOCATOR_PATH }}/track.yml --inventory-output ${{ env.ALLOCATOR_PATH }}/inventory.yml --instance-name gha_${{ github.run_id }}_ova_build \ + --label-team devops --label-termination-date 1d + sed 's/: */=/g' ${{ env.ALLOCATOR_PATH }}/inventory.yml > ${{ env.ALLOCATOR_PATH }}/inventory_mod.yml + sed -n 's/^identifier: \(.*\)$/identifier=\1/p' ${{ env.ALLOCATOR_PATH }}/track.yml >> ${{ env.ALLOCATOR_PATH }}/inventory_mod.yml + source ${{ env.ALLOCATOR_PATH }}/inventory_mod.yml + echo "::add-mask::$ansible_host" + echo "::add-mask::$ansible_port" + echo "::add-mask::$ansible_user" + echo "::add-mask::$ansible_ssh_private_key_file" + echo "::add-mask::$ansible_ssh_common_args" + echo "::add-mask::$identifier" + cat "${{ env.ALLOCATOR_PATH }}/inventory_mod.yml" >> $GITHUB_ENV; - - name: Wait for SSH to be available + - name: Generate inventory run: | - ansible_host=${{ env.ANSIBLE_HOST }} - MAX_RETRIES=40 - NUM_RETRIES=0 - while true; do - if sshpass -p ${{ env.OVA_USER_PASSWORD }} ssh -o 'StrictHostKeyChecking no' -o 'ConnectTimeout=10' ${{ env.OVA_USER }}@$ansible_host "exit"; then - break - fi - sleep 30 - NUM_RETRIES=$((NUM_RETRIES+1)) - if [ ${NUM_RETRIES} -eq ${MAX_RETRIES} ]; then - echo "Error connecting to OVA VM" - aws ec2 terminate-instances --instance-ids "${{ env.INSTANCE_ID }}" - exit 1 - fi - done + echo "[gha_instance]" > ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini + echo "${{ env.ansible_host }} ansible_port=${{ env.ansible_port }} ansible_user=${{ env.ansible_user }} ansible_ssh_private_key_file=${{ env.ansible_ssh_private_key_file }} ansible_ssh_common_args='${{ env.ansible_ssh_common_args }}'" >> ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini - - name: Install Python3 in the instance + - name: Install Python3 in the AWS instance run: | - sshpass -p ${{ env.OVA_USER_PASSWORD }} ssh -o 'StrictHostKeyChecking no' ${{ env.OVA_USER }}@${{ env.ANSIBLE_HOST }} "sudo yum install -y python3" + ssh -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" - name: Run Ansible playbook to generate the OVA run: | @@ -211,43 +183,17 @@ jobs: --extra-vars " \ wia_branch=${{ inputs.wazuh_installation_assistant_reference }} \ repository=${{ inputs.wazuh_package_repository }} \ + wvm_repository=${{ env.WVM_REPOSITORY }} \ + wvm_branch=${{ inputs.wazuh_virtual_machines_reference }} \ ova_path=${{ env.OVA_PATH }} \ wia_scripts=${{ env.WIA_DIR }} \ wia_repository=${{ env.WIA_REPOSITORY }} \ builder_args='$builder_args' \ debug=yes" ${{ inputs.debug }} - - name: Uninstall Python3 from the instance - run: | - sshpass -p ${{ env.OVA_USER_PASSWORD }} ssh -o 'StrictHostKeyChecking no' ${{ env.OVA_USER }}@${{ env.ANSIBLE_HOST }} "sudo yum remove -y python3" - - - name: Export Instance to create OVA - run: | - EXPORT=$(aws ec2 create-instance-export-task --instance-id "${{ env.INSTANCE_ID }}" --target-environment vmware \ - --export-to-s3-task "ContainerFormat=${{ env.CONTAINER_FORMAT }},DiskImageFormat=VMDK,S3Bucket=${{ env.TEMPORAL_S3_BUCKET }},S3Prefix=${{ env.TEMPORAL_S3_PATH }}/${{ env.FILENAME }}") - EXPORT_ID=$(echo ${EXPORT} | jq -r '.ExportTask.ExportTaskId') - echo "EXPORT_ID=${EXPORT_ID}" >> $GITHUB_ENV - - - name: Wait for export OVA + - name: Getting OVA from AWS instance run: | - MAX_RETRIES=67 - NUM_RETRIES=0 - while true; do - STATUS=$(aws ec2 describe-export-tasks --export-task-ids "${{ env.EXPORT_ID }}" | jq -r '.ExportTasks[0].State') - if [ "${STATUS}" == "completed" ]; then - break - fi - sleep 270 - NUM_RETRIES=$((NUM_RETRIES+1)) - if [ ${NUM_RETRIES} -eq ${MAX_RETRIES} ]; then - echo "Error exporting OVA" - exit 1 - fi - done - - - name: Getting OVA from temporal bucket - run: | - aws s3 --quiet cp "s3://${{ env.TEMPORAL_S3_BUCKET }}/${{ env.TEMPORAL_S3_PATH }}/${{ env.FILENAME }}${{ env.EXPORT_ID }}.ova" /tmp/${{ env.FILENAME_OVA }} + scp ${{ env.ansible_user }}@${{ env.ansible_host }}:/tmp/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} - name: Standarizing OVA run: | @@ -268,11 +214,6 @@ jobs: s3uri="s3://${{ env.S3_BUCKET }}/${{ env.S3_PATH }}/${{ env.FILENAME_SHA }}" echo "S3 sha512 OVA URI: ${s3uri}" - - name: Removing temporal files - run: | - aws s3 rm --quiet s3://${{ env.TEMPORAL_S3_BUCKET }}/${{ env.TEMPORAL_S3_PATH }}/${{ env.FILENAME }}${{ env.EXPORT_ID }}.ova - - name: Delete allocated VM - if: always() && steps.alloc_vm_ova.outcome == 'success' - run: | - aws ec2 terminate-instances --instance-ids "${{ env.INSTANCE_ID }}" + if: always() && steps.alloc_vm_ami.outcome == 'success' && inputs.destroy == true + run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml diff --git a/ova/workflow_assets/Vagrantfile b/ova/workflow_assets/Vagrantfile new file mode 100644 index 0000000..41818b6 --- /dev/null +++ b/ova/workflow_assets/Vagrantfile @@ -0,0 +1,92 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "al2023" + config.vm.synced_folder ".", "/vagrant/" + config.ssh.username = "wazuh-user" + config.ssh.password = "wazuh" + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # NOTE: This will enable public access to the opened port + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine and only allow access + # via 127.0.0.1 to disable public access + # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "192.168.56.22", interface: "2" + config.vm.hostname = "wazuh-server" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Disable the default share of the current code directory. Doing this + # provides improved isolation between the vagrant box and your host + # by making sure your Vagrantfile isn't accessible to the vagrant box. + # If you use this you may want to enable additional shared subfolders as + # shown above. + # config.vm.synced_folder ".", "/vagrant", disabled: true + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "8192" + vb.cpus = 4 + vb.customize ["setextradata", :id, "VBoxInternal/Devices/VMMDev/0/Config/GetHostTimeDisabled", 1] + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Enable provisioning with a shell script. Additional provisioners such as + # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # apt-get update + # apt-get install -y apache2 + # SHELL +# config.vm.provision "shell", inline: <<-SHELL +# sudo bash -c 'cat > /etc/systemd/network/20-enp0s8.network' << EOF +# [Match] +# Name=enp0s8 + +# [Network] +# DHCP=ipv4 +# EOF +# sudo systemctl restart systemd-networkd +# SHELL +end diff --git a/ova/workflow_assets/generate_base_box.sh b/ova/workflow_assets/generate_base_box.sh new file mode 100755 index 0000000..942b931 --- /dev/null +++ b/ova/workflow_assets/generate_base_box.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# AL2023 Vagrant base box generator +# Copyright (C) 2015, Wazuh Inc. +# +# This program is a free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public +# License (version 2) as published by the FSF - Free Software +# Foundation." + +set -euxo pipefail + +# Constants for version and filenames +AL2023_VERSION="latest" + +if [ "${AL2023_VERSION}" == "latest" ]; then + AL2023_VERSION=$(curl -I https://cdn.amazonlinux.com/al2023/os-images/latest/ | grep -i location | awk -F'/' '{print $(NF-1)}') +fi + +OVA_FILENAME="al2023-vmware_esx-${AL2023_VERSION}-kernel-6.1-x86_64.xfs.gpt.ova" +VMDK_FILENAME="al2023-vmware_esx-${AL2023_VERSION}-kernel-6.1-x86_64.xfs.gpt-disk1.vmdk" +AL2023_OVA_OUTPUT="al2023.ova" +# Temporary directories for raw, mount, and VDI files +RAW_DIR="$(mktemp -d -t al2023_raw_XXXXXXXX)" +MOUNT_DIR="$(mktemp -d -t al2023_mnt_XXXXXXXX)" +VDI_DIR="$(mktemp -d -t al2023_vdi_XXXXXXXX)" + +cleanup() { + # Cleanup temporary directories and unmount if necessary + umount "${MOUNT_DIR}/dev" || true + umount "${MOUNT_DIR}/proc" || true + umount "${MOUNT_DIR}/sys" || true + umount "${MOUNT_DIR}" || true + rm -rf "${RAW_DIR}" "${MOUNT_DIR}" "${VDI_DIR}" + vboxmanage unregistervm al2023 --delete || true +} + +trap cleanup EXIT + +check_dependencies() { + for cmd in vboxmanage wget tar chroot; do + if ! command -v "$cmd" &> /dev/null; then + echo "$cmd is required but not installed. Exiting." + exit 1 + fi + done +} + +download_and_extract_ova() { + if [ ! -f "${VMDK_FILENAME}" ]; then + wget "https://cdn.amazonlinux.com/al2023/os-images/${AL2023_VERSION}/vmware/${OVA_FILENAME}" + tar xvf "${OVA_FILENAME}" "${VMDK_FILENAME}" + fi +} + +convert_vmdk_to_raw() { + vboxmanage clonemedium "${VMDK_FILENAME}" "${RAW_DIR}/al2023.raw" --format RAW + vboxmanage closemedium "${VMDK_FILENAME}" + vboxmanage closemedium "${RAW_DIR}/al2023.raw" +} + +mount_and_setup_image() { + mount -o loop,offset=12582912 "${RAW_DIR}/al2023.raw" "${MOUNT_DIR}" + cp -a setup.sh "${MOUNT_DIR}/." + mount -o bind /dev "${MOUNT_DIR}/dev" + mount -o bind /proc "${MOUNT_DIR}/proc" + mount -o bind /sys "${MOUNT_DIR}/sys" + chroot "${MOUNT_DIR}" /setup.sh + umount "${MOUNT_DIR}/dev" + umount "${MOUNT_DIR}/proc" + umount "${MOUNT_DIR}/sys" + umount "${MOUNT_DIR}" +} + +convert_raw_to_vdi() { + vboxmanage convertfromraw "${RAW_DIR}/al2023.raw" "${VDI_DIR}/al2023.vdi" --format VDI +} + +create_virtualbox_vm() { + vboxmanage createvm --name al2023 --ostype Linux26_64 --register + vboxmanage modifyvm al2023 --memory 1024 --vram 16 --audio none + vboxmanage storagectl al2023 --name IDE --add ide + vboxmanage storagectl al2023 --name SATA --add sata --portcount 1 + vboxmanage storageattach al2023 --storagectl IDE --port 1 --device 0 --type dvddrive --medium emptydrive + vboxmanage storageattach al2023 --storagectl SATA --port 0 --device 0 --type hdd --medium "${VDI_DIR}/al2023.vdi" + if ! vboxmanage list hostonlyifs | grep -q "vboxnet0"; then + vboxmanage hostonlyif create + vboxmanage hostonlyif ipconfig vboxnet0 --ip 192.168.56.1 --netmask 255.255.255.0 + vboxmanage dhcpserver add --netname HostInterfaceNetworking-vboxnet0 \ + --ip 192.168.56.2 --netmask 255.255.255.0 \ + --lowerip 192.168.56.100 --upperip 192.168.56.200 --enable + fi + vboxmanage modifyvm al2023 --nic2 hostonly + vboxmanage modifyvm al2023 --hostonlyadapter2 vboxnet0 + vboxmanage modifyvm al2023 --cableconnected2 on +} + +package_vagrant_box() { + vagrant package --base al2023 --output al2023.box + vboxmanage export al2023 -o "${AL2023_OVA_OUTPUT}" +} + +# Main script execution +check_dependencies +download_and_extract_ova +convert_vmdk_to_raw +mount_and_setup_image +convert_raw_to_vdi +create_virtualbox_vm +package_vagrant_box \ No newline at end of file diff --git a/ova/workflow_assets/setup.sh b/ova/workflow_assets/setup.sh new file mode 100755 index 0000000..91fb32a --- /dev/null +++ b/ova/workflow_assets/setup.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Amazon Linux 2023 vagrant box construction, using an Amazon supplied VMDK +# disk image as a base. This script runs inside of a mounted Amazon Linux 2023 +# VMDK disk image, and sets up the vagrant related changes. + +# Greg Bailey +# November 25, 2023 + +set -eux + +# The image doesn't have any resolvers specified +configure_dns() { + rm -f /etc/resolv.conf + echo "nameserver 8.8.8.8" > /etc/resolv.conf +} + +# Set up wazuh-user +setup_user() { + useradd -m -s /bin/bash wazuh-user + echo "wazuh-user:wazuh" | chpasswd + + mkdir -p /home/wazuh-user/.ssh + wget -nv https://raw.githubusercontent.com/hashicorp/vagrant/main/keys/vagrant.pub -O /home/wazuh-user/.ssh/authorized_keys + chmod 600 /home/wazuh-user/.ssh/authorized_keys + chmod 700 /home/wazuh-user/.ssh + chown -R wazuh-user:wazuh-user /home/wazuh-user + + echo 'wazuh-user ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wazuh-user + chmod 440 /etc/sudoers.d/wazuh-user +} + +# Install legacy network-scripts required by Vagrant and git required to generate the OVA +install_dependencies() { + yum -y install network-scripts git +} + +# Install the VirtualBox guest additions +install_guest_additions() { + yum -y install gcc elfutils-libelf-devel kernel-devel libX11 libXt libXext libXmu + + dnf remove $(dnf repoquery --installonly --latest-limit=-1) + + KERNEL_VERSION=$(ls /lib/modules) + VIRTUALBOX_VERSION=$(wget -q http://download.virtualbox.org/virtualbox/LATEST.TXT -O -) + + wget -nv https://download.virtualbox.org/virtualbox/${VIRTUALBOX_VERSION}/VBoxGuestAdditions_${VIRTUALBOX_VERSION}.iso -O /root/VBoxGuestAdditions.iso + mount -o ro,loop /root/VBoxGuestAdditions.iso /mnt + sh /mnt/VBoxLinuxAdditions.run || true # Allow script to proceed despite potential errors + umount /mnt + rm -f /root/VBoxGuestAdditions.iso + + # Run VBox guest additions setup for the Amazon provided kernel + /etc/kernel/postinst.d/vboxadd ${KERNEL_VERSION} + /sbin/depmod ${KERNEL_VERSION} +} + +# Enable SSH password authentication +configure_ssh() { + sed -i 's/^#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config + systemctl restart sshd +} + + +# Clean up temporary files and free up space +cleanup() { + yum clean all + rm -rf /var/cache/yum/* + + rm -f /etc/resolv.conf + + rm -f /setup.sh + + for i in $(seq 2); do + sync + dd if=/dev/zero of=/zero$i bs=1M || true + sleep 1 + rm -f /zero$i + done +} + +# Main script execution +configure_dns +setup_user +install_dependencies +install_guest_additions +configure_ssh +cleanup \ No newline at end of file From ad73973fea8ec1d1dad134e07c18b166ad42b983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 16 Jan 2025 17:58:26 +0100 Subject: [PATCH 02/46] some changes in the playbook and adding of the python script file --- .../ansible_playbooks/ova_generator.yaml | 22 +++++++++++++++++++ .github/workflows/builder_OVA.yaml | 3 ++- ova/workflow_assets/Vagrantfile | 1 + ova/workflow_assets/ova_configurer.py | 0 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 ova/workflow_assets/ova_configurer.py diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 3371d1f..fbf7957 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -99,6 +99,28 @@ args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + - name: Execute Python script in the VM + shell: "vagrant ssh -c 'python3 /tmp/ova_directory/ova_configurer.py'" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + + - name: Stop the VM + shell: "vagrant halt" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + + # - name: Take the VM name + # shell: VBoxManage list vms | grep -o '"[^"]*"' | tr -d '"' + # register: vm_name_output + + - name: Configure VM network in VirtualBox + shell: | + vboxmanage modifyvm ova_base --nic2 hostonly + vboxmanage modifyvm ova_base --cableconnected2 on + + - name: Export the VM to OVA + shell: "vboxmanage export ova_base --output /home/ec2-user/{{ filename_ova }}" + # Migrate everything below to the Python script - name: Set custom hostname diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 4406ff7..5757f8d 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -188,12 +188,13 @@ jobs: ova_path=${{ env.OVA_PATH }} \ wia_scripts=${{ env.WIA_DIR }} \ wia_repository=${{ env.WIA_REPOSITORY }} \ + filename_ova=${{ env.FILENAME_OVA }} \ builder_args='$builder_args' \ debug=yes" ${{ inputs.debug }} - name: Getting OVA from AWS instance run: | - scp ${{ env.ansible_user }}@${{ env.ansible_host }}:/tmp/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} + scp ${{ env.ansible_user }}@${{ env.ansible_host }}:/home/ec2-user/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} - name: Standarizing OVA run: | diff --git a/ova/workflow_assets/Vagrantfile b/ova/workflow_assets/Vagrantfile index 41818b6..ff44ded 100644 --- a/ova/workflow_assets/Vagrantfile +++ b/ova/workflow_assets/Vagrantfile @@ -67,6 +67,7 @@ Vagrant.configure("2") do |config| vb.memory = "8192" vb.cpus = 4 vb.customize ["setextradata", :id, "VBoxInternal/Devices/VMMDev/0/Config/GetHostTimeDisabled", 1] + vb.name = "ova_base" end # # View the documentation for the provider you are using for more diff --git a/ova/workflow_assets/ova_configurer.py b/ova/workflow_assets/ova_configurer.py new file mode 100644 index 0000000..e69de29 From 704a5e88e77191ff5346bcc3da5aec2c9a8116b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Fri, 17 Jan 2025 17:11:58 +0100 Subject: [PATCH 03/46] created and developed the python script --- .../ansible_playbooks/ova_generator.yaml | 2 +- ova/workflow_assets/ova_configurer.py | 78 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index fbf7957..f0a2f1e 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -100,7 +100,7 @@ chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" - name: Execute Python script in the VM - shell: "vagrant ssh -c 'python3 /tmp/ova_directory/ova_configurer.py'" + shell: "vagrant ssh -c 'python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" diff --git a/ova/workflow_assets/ova_configurer.py b/ova/workflow_assets/ova_configurer.py index e69de29..1ddd406 100644 --- a/ova/workflow_assets/ova_configurer.py +++ b/ova/workflow_assets/ova_configurer.py @@ -0,0 +1,78 @@ +import os +import subprocess +import argparse + + +def set_hostname(): + """ + Sets the hostname of the machine + """ + subprocess.run("sudo hostnamectl set-hostname wazuh-server", shell=True, check=True) + +def install_git(): + """" + Installs git if it's not installed + """ + try: + subprocess.run("git --version", shell=True, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + subprocess.run("sudo yum install git -y", shell=True, check=True) + +def clone_repositories(): + """ + Clones the wazuh-installation-assistant and wazuh-virtual-machines repositories + """ + repos = [ + {"url": "https://github.com/wazuh/wazuh-virtual-machines.git", "dest": "/home/ec2-user/wazuh-virtual-machines"}, + {"url": "https://github.com/wazuh/wazuh-installation-assistant.git", "dest": "/home/ec2-user/wazuh-installation-assistant"} + ] + + for repo in repos: + subprocess.run(f"git clone {repo['url']} {repo['dest']}", shell=True, check=True) + + +def build_wazuh_install(repo_path, wia_branch): + """ + Builds the wazuh-install.sh script and moves it to /tmp + + Args: + repo_path (str): Local path of the repository + wia_branch (str): Branch of the wazuh-installation-assistant repository (version of Wazuh to install) + """ + + if os.path.exists(repo_path): + os.chdir(repo_path) + subprocess.run(f"git checkout {wia_branch}", shell=True, check=True) + subprocess.run("sudo bash builder.sh -i", shell=True, check=True) + if os.path.exists("wazuh-install.sh"): + subprocess.run("sudo mv wazuh-install.sh /tmp/wazuh-install.sh", shell=True, check=True) + + +def run_provision_script(repository, debug): + """ + Runs the provision.sh script + """ + os.chdir("/home/ec2-user/wazuh-virtual-machines/ova") + subprocess.run(f"sudo bash provision.sh {repository} {debug}", shell=True, check=True) + + +def main(): + """ + Main function + """ + parser = argparse.ArgumentParser() + parser.add_argument("--wia_branch", required=True, help="Branch of the wazuh-installation-assistant repository") + parser.add_argument("--repository", required=True, help="Production or development repository") + parser.add_argument("--debug", required=True, help="Debug mode") + args = parser.parse_args() + + set_hostname() + install_git() + clone_repositories() + build_wazuh_install("/home/ec2-user/wazuh-installation-assistant", args.wia_branch) + run_provision_script(args.repository, args.debug) + + +if __name__ == "__main__": + main() + From 17410b0910ac6de9cf173d92d6ba4ab927832325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 16:54:30 +0100 Subject: [PATCH 04/46] latests changes to the workflow --- .../ansible_playbooks/ova_generator.yaml | 73 +------------------ .github/workflows/builder_OVA.yaml | 6 +- ova/workflow_assets/ova_configurer.py | 68 +++++++++++++++-- 3 files changed, 66 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index f0a2f1e..1200327 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -100,7 +100,7 @@ chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" - name: Execute Python script in the VM - shell: "vagrant ssh -c 'python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" + shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" @@ -120,74 +120,3 @@ - name: Export the VM to OVA shell: "vboxmanage export ova_base --output /home/ec2-user/{{ filename_ova }}" - - # Migrate everything below to the Python script - - - name: Set custom hostname - command: "hostnamectl set-hostname wazuh-server" - - - name: Build Wazuh installation assistant script - command: "bash /tmp/{{ wia_scripts }}/builder.sh {{ builder_args }}" - - - name: Copy Wazuh installation assistant script to tmp dir - command: "cp /tmp/{{ wia_scripts }}/wazuh-install.sh /tmp/wazuh-install.sh" - - - name: Run provision script - command: "bash provision.sh {{ repository }} {{ debug }}" - args: - chdir: "{{ ova_path }}/ova" - async: 3600 - poll: 10 - - - name: Clean provision files - file: - path: /var/provision/ - state: absent - - - name: Clean Wazuh installation assistant resources - file: - path: /tmp/{{ wia_scripts }} - state: absent - - - name: Clean Wazuh installation assistant script - file: - path: /tmp/wazuh-install.sh - state: absent - - - name: Clean Wazuh installation assistant files - file: - path: /tmp/wazuh-install-files.tar - state: absent - - - name: Clean logs - shell: | - find /var/log/ -type f -exec bash -c 'cat /dev/null > {}' \; - find /var/ossec/logs -type f -execdir sh -c 'cat /dev/null > "$1"' _ {} \; - find /var/log/wazuh-indexer -type f -execdir sh -c 'cat /dev/null > "$1"' _ {} \; - find /var/log/filebeat -type f -execdir sh -c 'cat /dev/null > "$1"' _ {} \; - rm -rf /var/log/wazuh-install.log - - - name: Clean history - shell: cat /dev/null > ~/.bash_history && history -c - - - name: Clean YUM cache - shell: | - yum clean all - rm -rf /var/cache/yum/* - - - name: Remove AuthorizedKeysCommand from sshd_config - lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?AuthorizedKeysCommand.*' - state: absent - - - name: Remove AuthorizedKeysCommandUser from sshd_config - lineinfile: - path: /etc/ssh/sshd_config - regexp: '^#?AuthorizedKeysCommandUser.*' - state: absent - - - name: Restart SSH service - service: - name: sshd - state: restarted diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 5757f8d..a3a7699 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -215,6 +215,6 @@ jobs: s3uri="s3://${{ env.S3_BUCKET }}/${{ env.S3_PATH }}/${{ env.FILENAME_SHA }}" echo "S3 sha512 OVA URI: ${s3uri}" - - name: Delete allocated VM - if: always() && steps.alloc_vm_ami.outcome == 'success' && inputs.destroy == true - run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml + # - name: Delete allocated VM + # if: always() && steps.alloc_vm == 'success' && inputs.destroy == true + # run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml diff --git a/ova/workflow_assets/ova_configurer.py b/ova/workflow_assets/ova_configurer.py index 1ddd406..e39a65d 100644 --- a/ova/workflow_assets/ova_configurer.py +++ b/ova/workflow_assets/ova_configurer.py @@ -11,12 +11,9 @@ def set_hostname(): def install_git(): """" - Installs git if it's not installed + Installs git """ - try: - subprocess.run("git --version", shell=True, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - except subprocess.CalledProcessError: - subprocess.run("sudo yum install git -y", shell=True, check=True) + subprocess.run("sudo yum install git -y", shell=True, check=True) def clone_repositories(): """ @@ -51,10 +48,68 @@ def build_wazuh_install(repo_path, wia_branch): def run_provision_script(repository, debug): """ Runs the provision.sh script + + Args: + repository (str): Production or development repository + debug (str): Debug mode """ os.chdir("/home/ec2-user/wazuh-virtual-machines/ova") subprocess.run(f"sudo bash provision.sh {repository} {debug}", shell=True, check=True) + + +def create_network_config(): + """ + Creates the network configuration file and restarts the systemd-networkd service + """ + config_content = """[Match] +Name=eth1 +[Network] +DHCP=ipv4 +""" + + config_path = "/etc/systemd/network/20-eth1.network" + + with open(config_path, "w") as config_file: + config_file.write(config_content) + subprocess.run("sudo systemctl restart systemd-networkd", shell=True, check=True) + + +def clean(): + """ + Cleans the VM after the installation + """ + + os.remove("/tmp/wazuh-install.sh") + + subprocess.run("sudo rm -rf /home/ec2-user/wazuh-virtual-machines /home/ec2-user/wazuh-installation-assistant", shell=True, check=True) + + log_clean_commands = [ + "find /var/log/ -type f -exec bash -c 'cat /dev/null > {}' \\;", + "find /var/ossec/logs -type f -execdir sh -c 'cat /dev/null > \"$1\"' _ {} \\;", + "find /var/log/wazuh-indexer -type f -execdir sh -c 'cat /dev/null > \"$1\"' _ {} \;", + "find /var/log/filebeat -type f -execdir sh -c 'cat /dev/null > \"$1\"' _ {} \;", + "rm -rf /var/log/wazuh-install.log" + ] + for command in log_clean_commands: + subprocess.run(command, shell=True, check=True) + subprocess.run("cat /dev/null > ~/.bash_history && history -c", shell=True, check=True) + + yum_clean_commands = [ + "sudo yum clean all", + "sudo rm -rf /var/cache/yum/*" + ] + for command in yum_clean_commands: + subprocess.run(command, shell=True, check=True) + + sshd_config_changes = [ + (r'^#?AuthorizedKeysCommand.*', ''), + (r'^#?AuthorizedKeysCommandUser.*', ''), + ] + for pattern, replacement in sshd_config_changes: + subprocess.run(f"sudo sed -i '/{pattern}/d' /etc/ssh/sshd_config", shell=True, check=True) + subprocess.run("sudo systemctl restart sshd", shell=True, check=True) + def main(): """ @@ -71,7 +126,8 @@ def main(): clone_repositories() build_wazuh_install("/home/ec2-user/wazuh-installation-assistant", args.wia_branch) run_provision_script(args.repository, args.debug) - + create_network_config() + clean() if __name__ == "__main__": main() From a4561fb2048e47c6f6d81110caed05b6b728465c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 17:06:15 +0100 Subject: [PATCH 05/46] added amazon linux 2023 composite name --- .github/workflows/builder_OVA.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index a3a7699..b05b765 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -73,6 +73,7 @@ env: # TEMPORAL_S3_PATH: "trash/vm" # OVA_USER: "wazuh-user" # OVA_USER_PASSWORD: "wazuh" + COMPOSITE_NAME: "linux-amazon-2023-amd64" INVENTORY_PATH: "/tmp/allocatorvm_ova" AWS_REGION: "us-east-1" OVA_PATH: "/var/provision/wazuh-virtual-machines" From 1eae0e0422f9a5536c63eab9e4182676dc2744bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 17:14:20 +0100 Subject: [PATCH 06/46] added allocator path to the env --- .github/workflows/builder_OVA.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index b05b765..0a18963 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -74,7 +74,7 @@ env: # OVA_USER: "wazuh-user" # OVA_USER_PASSWORD: "wazuh" COMPOSITE_NAME: "linux-amazon-2023-amd64" - INVENTORY_PATH: "/tmp/allocatorvm_ova" + ALLOCATOR_PATH: "/tmp/allocatorvm_ova" AWS_REGION: "us-east-1" OVA_PATH: "/var/provision/wazuh-virtual-machines" WIA_DIR: "wazuh-installation-assistant" @@ -180,7 +180,7 @@ jobs: - name: Run Ansible playbook to generate the OVA run: | builder_args="-i" - ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook -i ${{ env.INVENTORY_PATH }}/inventory .github/workflows/ansible_playbooks/ova_generator.yaml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook -i ${{ env.ALLOCATOR_PATH }}/inventory .github/workflows/ansible_playbooks/ova_generator.yaml \ --extra-vars " \ wia_branch=${{ inputs.wazuh_installation_assistant_reference }} \ repository=${{ inputs.wazuh_package_repository }} \ From d4a8463b1485452a40ac7d58c8fcc8ad39e61e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 17:23:39 +0100 Subject: [PATCH 07/46] added port to ssh connection --- .github/workflows/builder_OVA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 0a18963..8cb33b9 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -175,7 +175,7 @@ jobs: - name: Install Python3 in the AWS instance run: | - ssh -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" + ssh -p ${{ env.ansible_port }} -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" - name: Run Ansible playbook to generate the OVA run: | From e951f8b44d0d9172e4d2b6d14bcf0002ceebba48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 17:28:56 +0100 Subject: [PATCH 08/46] added private key file to the ssh connection --- .github/workflows/builder_OVA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 8cb33b9..334cab1 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -175,7 +175,7 @@ jobs: - name: Install Python3 in the AWS instance run: | - ssh -p ${{ env.ansible_port }} -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" + ssh -p ${{ env.ansible_port }} -i ${{ env.ansible_ssh_private_key_file }} -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" - name: Run Ansible playbook to generate the OVA run: | From f479393184d25118d6919065de1d7dcad97af985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Mon, 20 Jan 2025 17:44:32 +0100 Subject: [PATCH 09/46] changed inventory path --- .github/workflows/builder_OVA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 334cab1..2d6d523 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -180,7 +180,7 @@ jobs: - name: Run Ansible playbook to generate the OVA run: | builder_args="-i" - ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook -i ${{ env.ALLOCATOR_PATH }}/inventory .github/workflows/ansible_playbooks/ova_generator.yaml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook -i ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini .github/workflows/ansible_playbooks/ova_generator.yaml \ --extra-vars " \ wia_branch=${{ inputs.wazuh_installation_assistant_reference }} \ repository=${{ inputs.wazuh_package_repository }} \ From 728f76f270912904d8870b5729dc03218ca7ff69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 10:46:43 +0100 Subject: [PATCH 10/46] added installation of aws collection --- .github/workflows/ansible_playbooks/ova_generator.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 1200327..e0af975 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -2,6 +2,9 @@ become: true tasks: + - name: Install AWS collection + run: ansible-galaxy collection install amazon.aws + - name: Enable Amazon Linux Extras repository amazon.aws.amazon_linux_extras: name: python3.8 From 3700620bd17a15d3e32959ac948e2f2cd07c6014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 10:53:16 +0100 Subject: [PATCH 11/46] fixed command in ansible playbook --- .github/workflows/ansible_playbooks/ova_generator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index e0af975..3cd009b 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -3,8 +3,8 @@ tasks: - name: Install AWS collection - run: ansible-galaxy collection install amazon.aws - + command: ansible-galaxy collection install amazon.aws + - name: Enable Amazon Linux Extras repository amazon.aws.amazon_linux_extras: name: python3.8 From c6b50f009ad054240986945ac4a01b72e0705972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 11:02:10 +0100 Subject: [PATCH 12/46] deleted libraries tasks in the playbook --- .github/workflows/ansible_playbooks/ova_generator.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 3cd009b..0a4d206 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -1,15 +1,7 @@ - hosts: all become: true - tasks: - - name: Install AWS collection - command: ansible-galaxy collection install amazon.aws - - - name: Enable Amazon Linux Extras repository - amazon.aws.amazon_linux_extras: - name: python3.8 - state: present - + tasks: - name: Install python3.8 and pip yum: name: From 8414930c0e3b5e5c20d05b497b21faeb2609bbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 11:28:48 +0100 Subject: [PATCH 13/46] removed python3 installation in the aws instance --- .../ansible_playbooks/ova_generator.yaml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 0a4d206..2d1d8f0 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -1,24 +1,17 @@ - hosts: all become: true - tasks: - - name: Install python3.8 and pip - yum: - name: - - python3.8 - - python3-pip - state: present - - - name: Configure Python 3.8 as default - alternatives: - name: python3 - path: /usr/bin/python3.8 - + tasks: - name: Update all the packages yum: name: '*' state: latest + - name: Install pip + yum: + name: python3-pip + state: present + - name: Install required tools for VirtualBox yum: name: From 3e9e8fb12aafec75aa61b4874bb08c7f19116658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 12:12:47 +0100 Subject: [PATCH 14/46] changed virtualbox and vagrant installation commands and added become in ansible tasks --- .../ansible_playbooks/ova_generator.yaml | 67 +++++++++---------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 2d1d8f0..27af15e 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -12,51 +12,48 @@ name: python3-pip state: present - - name: Install required tools for VirtualBox - yum: - name: - - gcc - - make - - perl - - bzip2 - - kernel-devel - - kernel-headers - - dkms - state: present + - name: Download VirtualBox installer script + get_url: + url: https://download.virtualbox.org/virtualbox/7.1.4/VirtualBox-7.1.4-165100-Linux_amd64.run + dest: /tmp/VirtualBox.run - - name: Add the VirtualBox repository - yum_repository: - name: virtualbox - description: "Oracle VirtualBox Repository" - baseurl: https://download.virtualbox.org/virtualbox/rpm/el/7/x86_64 - gpgcheck: yes - gpgkey: https://www.virtualbox.org/download/oracle_vbox.asc - enabled: yes + - name: Make the installer script executable + file: + path: /tmp/VirtualBox.run + mode: '0755' - - name: Install VirtualBox + - name: Run VirtualBox installer script + command: bash /tpm/VirtualBox.run + become: true + + - name: Update all the packages yum: - name: VirtualBox-7.0 - state: present + name: '*' + state: latest + + - name: Install Development tools + command: dnf groupinstall "Development Tools" -y + become: true + + - name: Install kernel headers + command: dnf install -y kernel-devel kernel-headers dkms elfutils-libelf-devel + become: true - name: Rebuild the VirtualBox kernel modules - command: "/sbin/vboxconfig" + command: /sbin/vboxconfig - - name: Add Vagrant repository - yum_repository: - name: vagrant - description: "HashiCorp Vagrant" - baseurl: https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo - gpgcheck: yes - gpgkey: https://rpm.releases.hashicorp.com/gpg - enabled: yes + - name: Install utilities for Vagrant + command: yum install -y yum-utils shadow-utils + + - name: Add the Vagrant repository + command: yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo - name: Install Vagrant - yum: - name: vagrant - state: present + command: yum -y install vagrant - name: Install git - shell: sudo yum install -y git + shell: yum install -y git + become: true - name: Create directory for the base VM file: From 42e5c2595a6e67fa45dcc1947c099ee34cac7b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 12:17:01 +0100 Subject: [PATCH 15/46] changed tpm for tmp typho --- .github/workflows/ansible_playbooks/ova_generator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 27af15e..bf39828 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -23,7 +23,7 @@ mode: '0755' - name: Run VirtualBox installer script - command: bash /tpm/VirtualBox.run + command: bash /tmp/VirtualBox.run become: true - name: Update all the packages From baf0d1c9c40f77de7972362635deaadecb3a28a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 12:44:40 +0100 Subject: [PATCH 16/46] installed dependencies for virtualbox installation --- .github/workflows/ansible_playbooks/ova_generator.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index bf39828..448db60 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -22,6 +22,11 @@ path: /tmp/VirtualBox.run mode: '0755' + - name: Install required packages for building kernel modules + yum: + name: kernel-devel kernel-headers dkms elfutils-libelf-devel gcc make perl + state: present + - name: Run VirtualBox installer script command: bash /tmp/VirtualBox.run become: true @@ -35,10 +40,6 @@ command: dnf groupinstall "Development Tools" -y become: true - - name: Install kernel headers - command: dnf install -y kernel-devel kernel-headers dkms elfutils-libelf-devel - become: true - - name: Rebuild the VirtualBox kernel modules command: /sbin/vboxconfig From d88ae30edfac8d1f0d3db9ff648a2285683807c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 12:50:17 +0100 Subject: [PATCH 17/46] separated dependencies in new lines --- .github/workflows/ansible_playbooks/ova_generator.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 448db60..5b88002 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -24,8 +24,16 @@ - name: Install required packages for building kernel modules yum: - name: kernel-devel kernel-headers dkms elfutils-libelf-devel gcc make perl + name: + - kernel-devel + - kernel-headers + - dkms + - elfutils-libelf-devel + - gcc + - make + - perl state: present + become: true - name: Run VirtualBox installer script command: bash /tmp/VirtualBox.run From bde34eab118241ce2acf4f46e555d4bd62824828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 13:16:38 +0100 Subject: [PATCH 18/46] changed tasks to move to the repository and checkout to the branch everytime in the playbook --- .../ansible_playbooks/ova_generator.yaml | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 5b88002..69cd60a 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -81,27 +81,35 @@ until: clone_result is success - name: Create base box - shell: "./generate_base_box.sh" - args: - chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + shell: | + cd /tmp/wazuh-virtual-machines && \ + git checkout {{ wvm_branch }} && \ + cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ + bash ./generate_base_box.sh - name: Add the created box shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_scripts/al2023.box" - name: Deploy the VM using Vagrant - shell: "vagrant up" - args: - chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + shell: | + cd /tmp/wazuh-virtual-machines && \ + git checkout {{ wvm_branch }} && \ + cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ + vagrant up - name: Execute Python script in the VM - shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" - args: - chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + shell: | + cd /tmp/wazuh-virtual-machines && \ + git checkout {{ wvm_branch }} && \ + cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ + "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" - name: Stop the VM - shell: "vagrant halt" - args: - chdir: "/tmp/wazuh-virtual-machines/ova/workflow_scripts" + shell: | + cd /tmp/wazuh-virtual-machines && \ + git checkout {{ wvm_branch }} && \ + cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ + vagrant halt # - name: Take the VM name # shell: VBoxManage list vms | grep -o '"[^"]*"' | tr -d '"' From c8ea638c55bd8a675eba7fdb80a40717d6913db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 13:24:51 +0100 Subject: [PATCH 19/46] changed the thypo from workflow_scripts to workflow_assets --- .../ansible_playbooks/ova_generator.yaml | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 69cd60a..c20c48b 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -81,35 +81,27 @@ until: clone_result is success - name: Create base box - shell: | - cd /tmp/wazuh-virtual-machines && \ - git checkout {{ wvm_branch }} && \ - cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ - bash ./generate_base_box.sh + shell: "./generate_base_box.sh" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - name: Add the created box - shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_scripts/al2023.box" + shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_assets/al2023.box" - name: Deploy the VM using Vagrant - shell: | - cd /tmp/wazuh-virtual-machines && \ - git checkout {{ wvm_branch }} && \ - cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ - vagrant up + shell: "vagrant up" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - name: Execute Python script in the VM - shell: | - cd /tmp/wazuh-virtual-machines && \ - git checkout {{ wvm_branch }} && \ - cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ - "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" + shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - name: Stop the VM - shell: | - cd /tmp/wazuh-virtual-machines && \ - git checkout {{ wvm_branch }} && \ - cd /tmp/wazuh-virtual-machines/ova/workflow_scripts && \ - vagrant halt + shell: "vagrant halt" + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" # - name: Take the VM name # shell: VBoxManage list vms | grep -o '"[^"]*"' | tr -d '"' From 09bb1245f54514cf80d31790ca07b46684204157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 13:46:48 +0100 Subject: [PATCH 20/46] add show output task when creating the base box and fixing scp command --- .github/workflows/ansible_playbooks/ova_generator.yaml | 6 ++++++ .github/workflows/builder_OVA.yaml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index c20c48b..10c0e65 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -84,6 +84,12 @@ shell: "./generate_base_box.sh" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + register: base_box_creation_result + + - name: Show the creation output + debug: + msg: "{{ base_box_creation_result.stdout }}" + msg: "{{ base_box_creation_result.stderr }}" - name: Add the created box shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_assets/al2023.box" diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 2d6d523..a871d04 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -195,7 +195,7 @@ jobs: - name: Getting OVA from AWS instance run: | - scp ${{ env.ansible_user }}@${{ env.ansible_host }}:/home/ec2-user/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} + scp -P ${{ env.ansible_port }} ${{ env.ansible_user }}@${{ env.ansible_host }}:/home/ec2-user/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} - name: Standarizing OVA run: | From f329085238280ec2bb03a80b2bb3e8a6f55ed24f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 14:53:56 +0100 Subject: [PATCH 21/46] added async and waiters to some tasks --- .../ansible_playbooks/ova_generator.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 10c0e65..a9b106b 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -85,12 +85,22 @@ args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" register: base_box_creation_result + async: 1800 + poll: 0 - name: Show the creation output debug: msg: "{{ base_box_creation_result.stdout }}" msg: "{{ base_box_creation_result.stderr }}" + - name: Wait for the base box creation to finish + async_status: + jid: "{{ base_box_creation_result.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + delay: 60 + - name: Add the created box shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_assets/al2023.box" @@ -98,11 +108,24 @@ shell: "vagrant up" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + retries: 10 + delay: 15 - name: Execute Python script in the VM shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + register: python_script_result + async: 1800 + poll: 0 + + - name: Wait for the Python script to finish + async_status: + jid: "{{ python_script_result.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + delay: 60 - name: Stop the VM shell: "vagrant halt" From 589655c82f8f8cf8fe7def5b84e12d53fa4cff62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 15:03:55 +0100 Subject: [PATCH 22/46] removed task for logs --- .github/workflows/ansible_playbooks/ova_generator.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index a9b106b..5ea08e0 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -88,11 +88,6 @@ async: 1800 poll: 0 - - name: Show the creation output - debug: - msg: "{{ base_box_creation_result.stdout }}" - msg: "{{ base_box_creation_result.stderr }}" - - name: Wait for the base box creation to finish async_status: jid: "{{ base_box_creation_result.ansible_job_id }}" From 5f2d8d1c78b401ca89fd77d2a0bca142441fda6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 17:13:06 +0100 Subject: [PATCH 23/46] added ignore errors when creating the base vagrant box --- .github/workflows/ansible_playbooks/ova_generator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 5ea08e0..6fa6a27 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -87,6 +87,7 @@ register: base_box_creation_result async: 1800 poll: 0 + ignore_errors: yes - name: Wait for the base box creation to finish async_status: From 8c9ad25ae615573c38af7ae1b9521684bb72856d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 18:25:56 +0100 Subject: [PATCH 24/46] deleted missconfiguration in the generate base box script --- ova/workflow_assets/generate_base_box.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ova/workflow_assets/generate_base_box.sh b/ova/workflow_assets/generate_base_box.sh index 942b931..f745298 100755 --- a/ova/workflow_assets/generate_base_box.sh +++ b/ova/workflow_assets/generate_base_box.sh @@ -83,16 +83,6 @@ create_virtualbox_vm() { vboxmanage storagectl al2023 --name SATA --add sata --portcount 1 vboxmanage storageattach al2023 --storagectl IDE --port 1 --device 0 --type dvddrive --medium emptydrive vboxmanage storageattach al2023 --storagectl SATA --port 0 --device 0 --type hdd --medium "${VDI_DIR}/al2023.vdi" - if ! vboxmanage list hostonlyifs | grep -q "vboxnet0"; then - vboxmanage hostonlyif create - vboxmanage hostonlyif ipconfig vboxnet0 --ip 192.168.56.1 --netmask 255.255.255.0 - vboxmanage dhcpserver add --netname HostInterfaceNetworking-vboxnet0 \ - --ip 192.168.56.2 --netmask 255.255.255.0 \ - --lowerip 192.168.56.100 --upperip 192.168.56.200 --enable - fi - vboxmanage modifyvm al2023 --nic2 hostonly - vboxmanage modifyvm al2023 --hostonlyadapter2 vboxnet0 - vboxmanage modifyvm al2023 --cableconnected2 on } package_vagrant_box() { From 2116226cd37b53f90be6f84662c8a48f87e2e096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Tue, 21 Jan 2025 18:52:51 +0100 Subject: [PATCH 25/46] added retries in the vagrant up task --- .github/workflows/ansible_playbooks/ova_generator.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 6fa6a27..5e23f54 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -104,8 +104,9 @@ shell: "vagrant up" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - retries: 10 - delay: 15 + retries: 20 + delay: 30 + until: result.rc == 0 - name: Execute Python script in the VM shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" From 858c8c69f480446165ce4739ad0d9e5fe9358aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 10:12:18 +0100 Subject: [PATCH 26/46] updated workflow to update the allocator directory to gh --- .github/workflows/builder_OVA.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index a871d04..2ca5f7f 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -173,6 +173,17 @@ jobs: echo "[gha_instance]" > ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini echo "${{ env.ansible_host }} ansible_port=${{ env.ansible_port }} ansible_user=${{ env.ansible_user }} ansible_ssh_private_key_file=${{ env.ansible_ssh_private_key_file }} ansible_ssh_common_args='${{ env.ansible_ssh_common_args }}'" >> ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini + - name: Compress Allocator directory + run: | + cd ${{ env.ALLOCATOR_PATH }} + zip -P "${{ secrets.ZIP_ARTIFACTS_PASSWORD }}" -r allocator_directory.zip . + + - name: Upload Allocator directory as artifact + uses: actions/upload-artifact@v4 + with: + name: allocator_directory + path: ${{ env.ALLOCATOR_PATH }}/allocator_directory.zip + - name: Install Python3 in the AWS instance run: | ssh -p ${{ env.ansible_port }} -i ${{ env.ansible_ssh_private_key_file }} -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" From 386f72b3815bc3333eeb50559c5d49d67790c24c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 11:38:36 +0100 Subject: [PATCH 27/46] added task to destroy previous machines --- .github/workflows/ansible_playbooks/ova_generator.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 5e23f54..b24e315 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -100,6 +100,16 @@ - name: Add the created box shell: "vagrant box add --name al2023 /tmp/wazuh-virtual-machines/ova/workflow_assets/al2023.box" + - name: Destroy previous machines + shell: | + cd /tmp/wazuh-virtual-machines/ova/workflow_assets + machines=$(vagrant global-status --prune | awk '/running|saved|poweroff/ {print $1}') + if [ -n "$machines" ]; then + for id in $machines; do + vagrant destroy -f $id + done + fi + - name: Deploy the VM using Vagrant shell: "vagrant up" args: From b9b53e2f7ae1bda68a1d1d695c422c3062aaefb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 12:19:53 +0100 Subject: [PATCH 28/46] added become to both tasks --- .github/workflows/ansible_playbooks/ova_generator.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index b24e315..ac28e1c 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -102,6 +102,7 @@ - name: Destroy previous machines shell: | + #!/bin/bash cd /tmp/wazuh-virtual-machines/ova/workflow_assets machines=$(vagrant global-status --prune | awk '/running|saved|poweroff/ {print $1}') if [ -n "$machines" ]; then @@ -109,6 +110,9 @@ vagrant destroy -f $id done fi + args: + executable: /bin/bash + become: true - name: Deploy the VM using Vagrant shell: "vagrant up" @@ -117,6 +121,7 @@ retries: 20 delay: 30 until: result.rc == 0 + become: true - name: Execute Python script in the VM shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" From 88e196677fbd7ecd949da264190b4d02363a2025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 12:47:28 +0100 Subject: [PATCH 29/46] change vagrant up command and added debug task --- .../ansible_playbooks/ova_generator.yaml | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index ac28e1c..e4cb8bf 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -114,15 +114,43 @@ executable: /bin/bash become: true - - name: Deploy the VM using Vagrant - shell: "vagrant up" + # - name: Deploy the VM using Vagrant + # shell: "vagrant up" + # args: + # chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + # retries: 20 + # delay: 30 + # until: result.rc == 0 + # become: true + + - name: Run vagrant up + shell: | + #!/bin/bash + MAX_RETRIES=20 + attempts=0 + cd /tmp/wazuh-virtual-machines/ova/workflow_assets + while true; do + ((attempts++)) + if vagrant up; then + break + else + if [ $attempts -eq $MAX_RETRIES ]; then + echo "Max attempts reached" + exit 1 + fi + vagrant destroy -f + fi + done args: - chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - retries: 20 - delay: 30 - until: result.rc == 0 + executable: /bin/bash + register: vagrant_up_result become: true - + + - name: Show the result of the vagrant up command + debug: + var: vagrant_up_result + verbosity: 2 + - name: Execute Python script in the VM shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" args: From ad1f974eedcc377e30a71b7fa28a1a7668dd297e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 13:29:19 +0100 Subject: [PATCH 30/46] add task to copy the python script to the vm --- .github/workflows/ansible_playbooks/ova_generator.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index e4cb8bf..3769deb 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -151,8 +151,16 @@ var: vagrant_up_result verbosity: 2 + - name: Copy the Python script to the VM + shell: | + vagrant plugin install vagrant-scp + vagrant scp ova_configurer.py ova_base:/tmp/ova_configurer.py + args: + chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + become: true + - name: Execute Python script in the VM - shell: "vagrant ssh -c 'sudo python3 /tmp/ova_directory/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" + shell: "vagrant ssh -c 'sudo python3 /tmp/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" register: python_script_result From 5bb4c4cacc8123fd79913dfd8dfc566ca7ad1f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 13:55:51 +0100 Subject: [PATCH 31/46] added more retries to vagrant up and an echo debug --- .github/workflows/ansible_playbooks/ova_generator.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 3769deb..10ebf43 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -126,11 +126,12 @@ - name: Run vagrant up shell: | #!/bin/bash - MAX_RETRIES=20 + MAX_RETRIES=30 attempts=0 cd /tmp/wazuh-virtual-machines/ova/workflow_assets while true; do ((attempts++)) + echo "Attempt $attempts" if vagrant up; then break else From e284255a569fdb88213da79f52987e3f5ab90d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 14:51:17 +0100 Subject: [PATCH 32/46] added block task --- .../ansible_playbooks/ova_generator.yaml | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 10ebf43..d4c5743 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -123,34 +123,37 @@ # until: result.rc == 0 # become: true - - name: Run vagrant up - shell: | - #!/bin/bash - MAX_RETRIES=30 - attempts=0 - cd /tmp/wazuh-virtual-machines/ova/workflow_assets - while true; do - ((attempts++)) - echo "Attempt $attempts" - if vagrant up; then - break - else - if [ $attempts -eq $MAX_RETRIES ]; then - echo "Max attempts reached" - exit 1 - fi - vagrant destroy -f - fi - done - args: - executable: /bin/bash - register: vagrant_up_result - become: true - - - name: Show the result of the vagrant up command - debug: - var: vagrant_up_result - verbosity: 2 + - name: Attempt to run vagrant up + block: + - name: Run vagrant up + shell: | + #!/bin/bash + MAX_RETRIES=30 + attempts=0 + cd /tmp/wazuh-virtual-machines/ova/workflow_assets + while true; do + ((attempts++)) + echo "Attempt $attempts" + if vagrant up; then + break + else + if [ $attempts -eq $MAX_RETRIES ]; then + echo "Max attempts reached" + exit 1 + fi + vagrant destroy -f + fi + done + args: + executable: /bin/bash + register: vagrant_up_result + become: true + + always: + - name: Show the result of the vagrant up command + debug: + var: vagrant_up_result + verbosity: 2 - name: Copy the Python script to the VM shell: | From 050e1df16f02605aab85e7bec9153903a4e0b1dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 15:32:43 +0100 Subject: [PATCH 33/46] redesing vagrant up task --- .../ansible_playbooks/ova_generator.yaml | 79 +++++++++++-------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index d4c5743..16590c3 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -110,8 +110,26 @@ vagrant destroy -f $id done fi + MAX_RETRIES=30 + attempts=0 + cd /tmp/wazuh-virtual-machines/ova/workflow_assets + while true; do + ((attempts++)) + echo "Attempt: $attempts" >&2 + if vagrant up; then + echo "vagrant up complete in attempt: $attempts" >&2 + break + else + if [ $attempts -eq $MAX_RETRIES ]; then + echo "Max attempts reached" >&2 + exit 1 + fi + vagrant destroy -f + fi + done args: executable: /bin/bash + register: vagrant_up_result become: true # - name: Deploy the VM using Vagrant @@ -123,37 +141,36 @@ # until: result.rc == 0 # become: true - - name: Attempt to run vagrant up - block: - - name: Run vagrant up - shell: | - #!/bin/bash - MAX_RETRIES=30 - attempts=0 - cd /tmp/wazuh-virtual-machines/ova/workflow_assets - while true; do - ((attempts++)) - echo "Attempt $attempts" - if vagrant up; then - break - else - if [ $attempts -eq $MAX_RETRIES ]; then - echo "Max attempts reached" - exit 1 - fi - vagrant destroy -f - fi - done - args: - executable: /bin/bash - register: vagrant_up_result - become: true - - always: - - name: Show the result of the vagrant up command - debug: - var: vagrant_up_result - verbosity: 2 + # - name: Attempt to run vagrant up + # block: + # - name: Run vagrant up + # shell: | + # #!/bin/bash + # MAX_RETRIES=30 + # attempts=0 + # cd /tmp/wazuh-virtual-machines/ova/workflow_assets + # while true; do + # ((attempts++)) + # echo "Attempt $attempts" + # if vagrant up; then + # break + # else + # if [ $attempts -eq $MAX_RETRIES ]; then + # echo "Max attempts reached" + # exit 1 + # fi + # vagrant destroy -f + # fi + # done + # args: + # executable: /bin/bash + # register: vagrant_up_result + # become: true + + - name: Show the result of the vagrant up command + debug: + var: vagrant_up_result + verbosity: 2 - name: Copy the Python script to the VM shell: | From 8388b475ebe1d467ee1f49fd7aed740347e69c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 17:02:18 +0100 Subject: [PATCH 34/46] added ignore error when vagrant up to see logs --- .github/workflows/ansible_playbooks/ova_generator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 16590c3..d918cda 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -131,6 +131,7 @@ executable: /bin/bash register: vagrant_up_result become: true + ignore_errors: yes # - name: Deploy the VM using Vagrant # shell: "vagrant up" From 43161df9965f7871d4abd929d85c40842e625e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 17:34:15 +0100 Subject: [PATCH 35/46] added async --- .../ansible_playbooks/ova_generator.yaml | 65 ++++++++----------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index d918cda..2a55717 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -110,18 +110,34 @@ vagrant destroy -f $id done fi + args: + executable: /bin/bash + register: vagrant_up_result + become: true + + # - name: Deploy the VM using Vagrant + # shell: "vagrant up" + # args: + # chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" + # retries: 20 + # delay: 30 + # until: result.rc == 0 + # become: true + + - name: Run vagrant up + shell: | + #!/bin/bash MAX_RETRIES=30 attempts=0 cd /tmp/wazuh-virtual-machines/ova/workflow_assets while true; do ((attempts++)) - echo "Attempt: $attempts" >&2 + echo "Attempt $attempts" if vagrant up; then - echo "vagrant up complete in attempt: $attempts" >&2 break else if [ $attempts -eq $MAX_RETRIES ]; then - echo "Max attempts reached" >&2 + echo "Max attempts reached" exit 1 fi vagrant destroy -f @@ -129,44 +145,19 @@ done args: executable: /bin/bash + async: 1800 + poll: 0 register: vagrant_up_result become: true - ignore_errors: yes - # - name: Deploy the VM using Vagrant - # shell: "vagrant up" - # args: - # chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - # retries: 20 - # delay: 30 - # until: result.rc == 0 - # become: true + - name: Wait for vagrant up to finish + async_status: + jid: "{{ vagrant_up_result.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + delay: 60 - # - name: Attempt to run vagrant up - # block: - # - name: Run vagrant up - # shell: | - # #!/bin/bash - # MAX_RETRIES=30 - # attempts=0 - # cd /tmp/wazuh-virtual-machines/ova/workflow_assets - # while true; do - # ((attempts++)) - # echo "Attempt $attempts" - # if vagrant up; then - # break - # else - # if [ $attempts -eq $MAX_RETRIES ]; then - # echo "Max attempts reached" - # exit 1 - # fi - # vagrant destroy -f - # fi - # done - # args: - # executable: /bin/bash - # register: vagrant_up_result - # become: true - name: Show the result of the vagrant up command debug: From a2252a8f5d192d9ecbd963115cb4bbf75da06443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Wed, 22 Jan 2025 18:17:37 +0100 Subject: [PATCH 36/46] fixed scp command --- .github/workflows/ansible_playbooks/ova_generator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 2a55717..15b0f4d 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -158,7 +158,6 @@ retries: 30 delay: 60 - - name: Show the result of the vagrant up command debug: var: vagrant_up_result @@ -166,8 +165,9 @@ - name: Copy the Python script to the VM shell: | + cd /tmp/wazuh-virtual-machines/ova/workflow_assets vagrant plugin install vagrant-scp - vagrant scp ova_configurer.py ova_base:/tmp/ova_configurer.py + vagrant scp ova_configurer.py :/tmp/ova_configurer.py args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" become: true From 6c8a68c663e63f03f864a289846518905b203f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 10:55:25 +0100 Subject: [PATCH 37/46] added change of the wvm repository --- .github/workflows/ansible_playbooks/ova_generator.yaml | 2 +- ova/workflow_assets/ova_configurer.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 15b0f4d..8477f3f 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -173,7 +173,7 @@ become: true - name: Execute Python script in the VM - shell: "vagrant ssh -c 'sudo python3 /tmp/ova_configurer.py --wia_branch {{ wia_branch }} --repository {{ repository }} --debug {{ debug}}'" + shell: "vagrant ssh -c 'sudo python3 /tmp/ova_configurer.py --wia_branch {{ wia_branch }} --wvm_branch {{ wvm_branch }} --repository {{ repository }} --debug {{ debug}}'" args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" register: python_script_result diff --git a/ova/workflow_assets/ova_configurer.py b/ova/workflow_assets/ova_configurer.py index e39a65d..3e7abe0 100644 --- a/ova/workflow_assets/ova_configurer.py +++ b/ova/workflow_assets/ova_configurer.py @@ -45,7 +45,7 @@ def build_wazuh_install(repo_path, wia_branch): subprocess.run("sudo mv wazuh-install.sh /tmp/wazuh-install.sh", shell=True, check=True) -def run_provision_script(repository, debug): +def run_provision_script(wvm_branch, repository, debug): """ Runs the provision.sh script @@ -54,6 +54,7 @@ def run_provision_script(repository, debug): debug (str): Debug mode """ os.chdir("/home/ec2-user/wazuh-virtual-machines/ova") + subprocess.run(f"git checkout {wvm_branch}", shell=True, check=True) subprocess.run(f"sudo bash provision.sh {repository} {debug}", shell=True, check=True) @@ -117,6 +118,7 @@ def main(): """ parser = argparse.ArgumentParser() parser.add_argument("--wia_branch", required=True, help="Branch of the wazuh-installation-assistant repository") + parser.add_argument("--wvm_branch", required=True, help="Branch of the wazuh-virtual-machines repository") parser.add_argument("--repository", required=True, help="Production or development repository") parser.add_argument("--debug", required=True, help="Debug mode") args = parser.parse_args() @@ -125,7 +127,7 @@ def main(): install_git() clone_repositories() build_wazuh_install("/home/ec2-user/wazuh-installation-assistant", args.wia_branch) - run_provision_script(args.repository, args.debug) + run_provision_script(args.wvm_branch, args.repository, args.debug) create_network_config() clean() From c66c4b0ddec490e881e4f6f7f0e599f58ff57ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 11:36:00 +0100 Subject: [PATCH 38/46] added async to the export ova task --- .../workflows/ansible_playbooks/ova_generator.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 8477f3f..d760334 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -204,3 +204,14 @@ - name: Export the VM to OVA shell: "vboxmanage export ova_base --output /home/ec2-user/{{ filename_ova }}" + register: export_result + async: 1800 + poll: 0 + + - name: Wait for export the OVA + async_status: + jid: "{{ export_result.ansible_job_id }}" + register: job_result + until: job_result.finished + retries: 30 + delay: 60 From ba62cd83b9a4b29ab872a6d00cceb7291c3b4840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 11:42:37 +0100 Subject: [PATCH 39/46] commented dev or prod in the provision.sh --- ova/provision.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ova/provision.sh b/ova/provision.sh index b4118fa..20d9234 100755 --- a/ova/provision.sh +++ b/ova/provision.sh @@ -13,11 +13,11 @@ ASSETS_PATH="${CURRENT_PATH}/assets" CUSTOM_PATH="${ASSETS_PATH}/custom" INSTALL_ARGS="-a" -if [[ "${PACKAGES_REPOSITORY}" == "dev" ]]; then - INSTALL_ARGS+=" -d pre-release" -elif [[ "${PACKAGES_REPOSITORY}" == "staging" ]]; then - INSTALL_ARGS+=" -d staging" -fi +# if [[ "${PACKAGES_REPOSITORY}" == "dev" ]]; then +# INSTALL_ARGS+=" -d pre-release" +# elif [[ "${PACKAGES_REPOSITORY}" == "staging" ]]; then +# INSTALL_ARGS+=" -d staging" +# fi if [[ "${DEBUG}" = "yes" ]]; then INSTALL_ARGS+=" -v" From 3d49271b832ca12b16e524d81739f7c54e3f09cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 12:25:02 +0100 Subject: [PATCH 40/46] added private key file to the scp task --- .github/workflows/builder_OVA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 2ca5f7f..0008e45 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -206,7 +206,7 @@ jobs: - name: Getting OVA from AWS instance run: | - scp -P ${{ env.ansible_port }} ${{ env.ansible_user }}@${{ env.ansible_host }}:/home/ec2-user/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} + scp -P ${{ env.ansible_port }} -i ${{ env.ansible_ssh_private_key_file }} ${{ env.ansible_user }}@${{ env.ansible_host }}:/home/ec2-user/${{ env.FILENAME_OVA }} /tmp/${{ env.FILENAME_OVA }} - name: Standarizing OVA run: | From 5989503e796bb4fec32aec911b455d11d6af918a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 13:12:14 +0100 Subject: [PATCH 41/46] added permissions to the exported ova file --- .github/workflows/ansible_playbooks/ova_generator.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index d760334..7ffa887 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -215,3 +215,8 @@ until: job_result.finished retries: 30 delay: 60 + + - name: Change permissions to the OVA + file: + path: /home/ec2-user/{{ filename_ova }} + mode: '0755' From 23b57c07a45af57ef72d84ab40dde20e92025d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 13:28:53 +0100 Subject: [PATCH 42/46] raised the retries for the vagrant up task --- .github/workflows/ansible_playbooks/ova_generator.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index 7ffa887..c1385e9 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -127,7 +127,7 @@ - name: Run vagrant up shell: | #!/bin/bash - MAX_RETRIES=30 + MAX_RETRIES=100 attempts=0 cd /tmp/wazuh-virtual-machines/ova/workflow_assets while true; do @@ -145,7 +145,7 @@ done args: executable: /bin/bash - async: 1800 + async: 7200 poll: 0 register: vagrant_up_result become: true @@ -155,7 +155,7 @@ jid: "{{ vagrant_up_result.ansible_job_id }}" register: job_result until: job_result.finished - retries: 30 + retries: 120 delay: 60 - name: Show the result of the vagrant up command From ad3861245a6a5c4eae2d6319a8f8a7d219e74e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 16:40:05 +0100 Subject: [PATCH 43/46] last changes to the ova workflow files deleting comments and changing filename of the ova --- .../ansible_playbooks/ova_generator.yaml | 15 +---- .github/workflows/builder_OVA.yaml | 33 +++------- ova/provision.sh | 6 -- ova/workflow_assets/Vagrantfile | 64 ------------------- 4 files changed, 10 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ansible_playbooks/ova_generator.yaml b/.github/workflows/ansible_playbooks/ova_generator.yaml index c1385e9..bca8f81 100644 --- a/.github/workflows/ansible_playbooks/ova_generator.yaml +++ b/.github/workflows/ansible_playbooks/ova_generator.yaml @@ -115,15 +115,6 @@ register: vagrant_up_result become: true - # - name: Deploy the VM using Vagrant - # shell: "vagrant up" - # args: - # chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - # retries: 20 - # delay: 30 - # until: result.rc == 0 - # become: true - - name: Run vagrant up shell: | #!/bin/bash @@ -193,10 +184,6 @@ args: chdir: "/tmp/wazuh-virtual-machines/ova/workflow_assets" - # - name: Take the VM name - # shell: VBoxManage list vms | grep -o '"[^"]*"' | tr -d '"' - # register: vm_name_output - - name: Configure VM network in VirtualBox shell: | vboxmanage modifyvm ova_base --nic2 hostonly @@ -216,7 +203,7 @@ retries: 30 delay: 60 - - name: Change permissions to the OVA + - name: Change permissions to the OVA file file: path: /home/ec2-user/{{ filename_ova }} mode: '0755' diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 0008e45..4a86775 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -61,18 +61,9 @@ on: required: false env: - # OVA_AMI: "ami-0d4bd55523ee67aa4" INSTANCE_TYPE: "metal" - # SECURITY_GROUP: "sg-005cff996b335d497" - # SUBNET: "subnet-0b6aea31fb32cffad" - # TEMPORAL_S3_BUCKET: "warehouse.wazuh.com" S3_BUCKET: ${{ vars.AWS_S3_BUCKET }} S3_PATH: "development/wazuh/4.x/secondary/ova" - # OVA_ENVIRONMENT: "vmware" - # CONTAINER_FORMAT: "ova" - # TEMPORAL_S3_PATH: "trash/vm" - # OVA_USER: "wazuh-user" - # OVA_USER_PASSWORD: "wazuh" COMPOSITE_NAME: "linux-amazon-2023-amd64" ALLOCATOR_PATH: "/tmp/allocatorvm_ova" AWS_REGION: "us-east-1" @@ -114,9 +105,14 @@ jobs: with: ref: ${{ inputs.wazuh_virtual_machines_reference }} + - name: Checkout wazuh/wazuh-installation-assistant repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.wazuh_installation_assistant_reference }} + - name: Setting FILENAME var run: | - WAZUH_VERSION=$(cat VERSION) + WAZUH_VERSION=$(cat wazuh-installation-assistant/VERSION) COMMIT_SHA=$(git rev-parse --short ${{ github.sha }}) echo "WAZUH_VERSION=$WAZUH_VERSION" >> $GITHUB_ENV FILENAME="wazuh-${WAZUH_VERSION}-${{ inputs.ova_revision }}" @@ -173,17 +169,6 @@ jobs: echo "[gha_instance]" > ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini echo "${{ env.ansible_host }} ansible_port=${{ env.ansible_port }} ansible_user=${{ env.ansible_user }} ansible_ssh_private_key_file=${{ env.ansible_ssh_private_key_file }} ansible_ssh_common_args='${{ env.ansible_ssh_common_args }}'" >> ${{ env.ALLOCATOR_PATH }}/inventory_ansible.ini - - name: Compress Allocator directory - run: | - cd ${{ env.ALLOCATOR_PATH }} - zip -P "${{ secrets.ZIP_ARTIFACTS_PASSWORD }}" -r allocator_directory.zip . - - - name: Upload Allocator directory as artifact - uses: actions/upload-artifact@v4 - with: - name: allocator_directory - path: ${{ env.ALLOCATOR_PATH }}/allocator_directory.zip - - name: Install Python3 in the AWS instance run: | ssh -p ${{ env.ansible_port }} -i ${{ env.ansible_ssh_private_key_file }} -o 'StrictHostKeyChecking no' ${{ env.ansible_user }}@${{ env.ansible_host }} "sudo yum install -y python3" @@ -227,6 +212,6 @@ jobs: s3uri="s3://${{ env.S3_BUCKET }}/${{ env.S3_PATH }}/${{ env.FILENAME_SHA }}" echo "S3 sha512 OVA URI: ${s3uri}" - # - name: Delete allocated VM - # if: always() && steps.alloc_vm == 'success' && inputs.destroy == true - # run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml + - name: Delete allocated VM + if: always() && steps.alloc_vm == 'success' && inputs.destroy == true + run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml diff --git a/ova/provision.sh b/ova/provision.sh index 20d9234..e91f873 100755 --- a/ova/provision.sh +++ b/ova/provision.sh @@ -13,12 +13,6 @@ ASSETS_PATH="${CURRENT_PATH}/assets" CUSTOM_PATH="${ASSETS_PATH}/custom" INSTALL_ARGS="-a" -# if [[ "${PACKAGES_REPOSITORY}" == "dev" ]]; then -# INSTALL_ARGS+=" -d pre-release" -# elif [[ "${PACKAGES_REPOSITORY}" == "staging" ]]; then -# INSTALL_ARGS+=" -d staging" -# fi - if [[ "${DEBUG}" = "yes" ]]; then INSTALL_ARGS+=" -v" fi diff --git a/ova/workflow_assets/Vagrantfile b/ova/workflow_assets/Vagrantfile index ff44ded..5a79d08 100644 --- a/ova/workflow_assets/Vagrantfile +++ b/ova/workflow_assets/Vagrantfile @@ -6,63 +6,19 @@ # backwards compatibility). Please don't change it unless you know what # you're doing. Vagrant.configure("2") do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - # Every Vagrant development environment requires a box. You can search for - # boxes at https://vagrantcloud.com/search. config.vm.box = "al2023" config.vm.synced_folder ".", "/vagrant/" config.ssh.username = "wazuh-user" config.ssh.password = "wazuh" - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # NOTE: This will enable public access to the opened port - # config.vm.network "forwarded_port", guest: 80, host: 8080 - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine and only allow access - # via 127.0.0.1 to disable public access - # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" # Create a private network, which allows host-only access to the machine # using a specific IP. config.vm.network "private_network", ip: "192.168.56.22", interface: "2" config.vm.hostname = "wazuh-server" - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" - - # Disable the default share of the current code directory. Doing this - # provides improved isolation between the vagrant box and your host - # by making sure your Vagrantfile isn't accessible to the vagrant box. - # If you use this you may want to enable additional shared subfolders as - # shown above. - # config.vm.synced_folder ".", "/vagrant", disabled: true - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. # Example for VirtualBox: # config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # # # Customize the amount of memory on the VM: vb.memory = "8192" vb.cpus = 4 @@ -70,24 +26,4 @@ Vagrant.configure("2") do |config| vb.name = "ova_base" end # - # View the documentation for the provider you are using for more - # information on available options. - - # Enable provisioning with a shell script. Additional provisioners such as - # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the - # documentation for more information about their specific syntax and use. - # config.vm.provision "shell", inline: <<-SHELL - # apt-get update - # apt-get install -y apache2 - # SHELL -# config.vm.provision "shell", inline: <<-SHELL -# sudo bash -c 'cat > /etc/systemd/network/20-enp0s8.network' << EOF -# [Match] -# Name=enp0s8 - -# [Network] -# DHCP=ipv4 -# EOF -# sudo systemctl restart systemd-networkd -# SHELL end From d99d5e9acb08d10461a70dd99c836af541e7f4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 16:55:00 +0100 Subject: [PATCH 44/46] fixed clone of the wazuh-installation-assistant repo --- .github/workflows/builder_OVA.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 4a86775..2a25de4 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -108,6 +108,7 @@ jobs: - name: Checkout wazuh/wazuh-installation-assistant repository uses: actions/checkout@v4 with: + repository: wazuh/wazuh-installation-assistant ref: ${{ inputs.wazuh_installation_assistant_reference }} - name: Setting FILENAME var From 25294e69d613923b99d1c2627f14142927b4db21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 16:57:11 +0100 Subject: [PATCH 45/46] fix clone of wazuh-installation-assistant repo --- .github/workflows/builder_OVA.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index 2a25de4..c950b0b 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -110,6 +110,8 @@ jobs: with: repository: wazuh/wazuh-installation-assistant ref: ${{ inputs.wazuh_installation_assistant_reference }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-installation-assistant - name: Setting FILENAME var run: | From 34f3ae734f355671ab8f519f300bd349ae832f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Anguita=20L=C3=B3pez?= Date: Thu, 23 Jan 2025 17:52:56 +0100 Subject: [PATCH 46/46] fixed destroy of the aws instance --- .github/workflows/builder_OVA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/builder_OVA.yaml b/.github/workflows/builder_OVA.yaml index c950b0b..8c64bb6 100644 --- a/.github/workflows/builder_OVA.yaml +++ b/.github/workflows/builder_OVA.yaml @@ -216,5 +216,5 @@ jobs: echo "S3 sha512 OVA URI: ${s3uri}" - name: Delete allocated VM - if: always() && steps.alloc_vm == 'success' && inputs.destroy == true + if: always() && steps.alloc_vm == 'success' run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output ${{ env.ALLOCATOR_PATH }}/track.yml