From d3b5b3d2d2705fb464117cfaf14928b3052f4d26 Mon Sep 17 00:00:00 2001 From: William Stearns Date: Wed, 14 Aug 2024 15:02:15 -0400 Subject: [PATCH] Split prep and post tasks into their own playbook (#16) * Split prep and post tasks into their own playbook * all playbooks in one command * Include install_pre and install_post playbooks in installer generator --------- Co-authored-by: Naomi Kramer --- installer/Installer.md | 59 ++- installer/generate_installer.sh | 6 +- installer/install_scripts/install_post.yml | 63 +++ installer/install_scripts/install_pre.yml | 494 +++++++++++++++++++ installer/install_scripts/install_rita.sh | 11 +- installer/install_scripts/install_rita.yml | 532 +-------------------- installer/install_scripts/install_zeek.yml | 522 +------------------- 7 files changed, 649 insertions(+), 1038 deletions(-) create mode 100644 installer/install_scripts/install_post.yml create mode 100644 installer/install_scripts/install_pre.yml diff --git a/installer/Installer.md b/installer/Installer.md index f8b4009..3da43b5 100644 --- a/installer/Installer.md +++ b/installer/Installer.md @@ -1,7 +1,13 @@ +## RITA/Zeek Installer + +#### Generated installer directory ``` rita-.tar.gz │ install_rita.yml │ install_rita.sh +| install_zeek.yml +| install_pre.yml +| install_post.yml │ └───/scripts │ │ ansible-installer.sh @@ -19,6 +25,7 @@ rita-.tar.gz │ │ │ README │ │ │ LICENSE │ │ │ rita.sh +| | | zeek │ │ │ └───/etc │ │ config.hjson @@ -29,4 +36,54 @@ rita-.tar.gz │ │ timezone.xml │ └───/threat_intel_feeds -``` \ No newline at end of file +``` + + +### Generating an installer + +Note: generating the installer on a branch that has no tag when running `git describe --always --abbrev=0 --tags` will generate a broken installer. + +Run: +`./installer/generate_installer.sh` + +The script will generate an installer tar file in the `installer` folder, named `rita-v-installer.tar.gz`. + +Verify that all files in the above directory tree exist in the generated tar file. + +Verify that all occurences of "REPLACE_ME" within scripts and/or playbooks got updated with the proper version number that is expected. +The version for RITA that gets replaced should match the current tag. + +The version for Zeek that gets replaced should be the desired version of docker-zeek to be used in this release. + +The docker-zeek repo pushes a built multi-architecture image of zeek to DockerHub using Github Actions. The generate_installer script should specify which tag version on [Dockerhub](https://hub.docker.com/r/activecm/zeek/tags) you wish to include with this release. Multi-architecture tags require all architectures to finish building before being merged into one tag, so if the build actions are in progress, please be patient and wait for them to finish before attempting to install it. + +### Running the installer +To install RITA on the current system, run: +`./rita-v-installer/install_rita.sh localhost`. + +To install RITA on a remote system, run: +`./rita-v-installer/install_rita.sh root@8.8.8.8`. + +### Updating the installer +Each file that is expected to be in the installer must be explicitly copied to the installer within the `./installer/generate_installer.sh` script. + +If any new Ansible playbook or script that uses the "REPLACE_ME" string to insert a version is added, the generate_installer script must be updated to replace that string with the proper version. + +Any versions for RITA should NOT be hard-coded. The version should be retrieved by the generate_installer script automatically. The only hard-coded versions in the generator should be for external projects. + + +### "One-line installer" +To make installing both RITA and Zeek easier, a one-line installer is created and uploaded to the release artifacts on Github. This installer is generated with the generate_installer.sh script as well, but is uploaded to the release within the Generate Installer Github Action. +This one line installer is a single script (not a tar file). It installs RITA & Zeek on the local system and does NOT require passing any arguments to it. + + +### Zeek +There are multiple moving parts in order to build Zeek and include it in a RITA install bundle. + +The main Zeek repo is [docker-zeek](https://github.com/activecm/docker-zeek). This repository contains the Dockerfile definition needed to build the docker image of Zeek that includes custom modifications like timeouts and the [zeek-open-connections](https://github.com/activecm/zeek-open-connections) plugin. + +The docker-zeek repo is responsible for building the multi-arch image for Zeek in Github Actions. The actions automatically upload the image to Dockerhub. In order to test changes locally without uploading them to Dockerhub, the docker-zeek image must be built on your local system and tagged with a name that is NOT similar to `activecm/zeek:`. To test the zeek script with this custom-built image, the `zeek` script in the docker-zeek repo must be updated to use your custom tag instead of whatever is listed in the `IMAGE_NAME` variable. + +The zeek-open-connections plugin must have an updated tag in order to be recognized by the Zeek package manager (zkg). Follow the instructions in that repo's README for more details. + +The RITA installer includes an Ansible playbook that pulls the desired version of `activecm/zeek` from Dockerhub and creates the necessary directories needed to run Zeek. The installer generator also pulls the `zeek` script from the `docker-zeek` repo and includes it in the installer, along with listing the proper image version in the `IMAGE_NAME` variable. Aside from these two items, Zeek and RITA are independent of each other. \ No newline at end of file diff --git a/installer/generate_installer.sh b/installer/generate_installer.sh index 94d0034..8171751 100755 --- a/installer/generate_installer.sh +++ b/installer/generate_installer.sh @@ -22,15 +22,12 @@ rm -rf "$BASE_DIR" # mkdir ./stage # create ansible subfolders -# ANSIBLE_FILES=./stage/.ansible/files SCRIPTS="$BASE_DIR/scripts" ANSIBLE_FILES="$BASE_DIR/files" -ANSIBLE_PLAYBOOKS="$BASE_DIR/.ansible/playbooks" mkdir "$BASE_DIR" mkdir -p "$ANSIBLE_FILES" mkdir -p "$SCRIPTS" -mkdir -p "$ANSIBLE_PLAYBOOKS" # create subfolders (for files that installed RITA will contain) INSTALL_OPT="$ANSIBLE_FILES"/opt @@ -42,6 +39,9 @@ mkdir "$ANSIBLE_FILES"/etc # copy files in base dir cp ./install_scripts/install_zeek.yml "$BASE_DIR" cp ./install_scripts/install_rita.yml "$BASE_DIR" +cp ./install_scripts/install_pre.yml "$BASE_DIR" +cp ./install_scripts/install_post.yml "$BASE_DIR" + cp ./install_scripts/install_rita.sh "$BASE_DIR" # entrypoint # copy files to helper script folder diff --git a/installer/install_scripts/install_post.yml b/installer/install_scripts/install_post.yml new file mode 100644 index 0000000..ba60512 --- /dev/null +++ b/installer/install_scripts/install_post.yml @@ -0,0 +1,63 @@ +--- +# ansible install playbook for rita V2. +# Version: 202408061413 +# sample runs: +# Optional: Add the following block, without #'s to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports). +# The hosts must each be on their own line. These can be full or short hostnames or a name following "Host" in ~/.ssh/config . +# +# [allritas] +# ro810 +# ub2404 +# +# Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: +# +# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run +# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' + +# Many thanks to but-i-am-dominator for his help with this playbook. + + +- name: "RITA Post: Reboot checks." + hosts: "{{ install_hosts }}" + become: true + + vars: + ansible_python_interpreter: /bin/python3 # Centos 7 defaults to using python2, so we force python 3. This change does not break any other distros + + +#Late tasks, including rebooting + post_tasks: + - name: "RITA Post: Check if reboot required on rpm-based systems." + command: needs-restarting -r + register: reboot_result + ignore_errors: true + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + tags: + - packages + - linux + - linuxrpm + + - name: "RITA Post: Check if reboot required on Debian/Ubuntu-based systems." + register: reboot_required_file + stat: + path: /var/run/reboot-required + get_checksum: no + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Post: Rebooting system if needed." + reboot: + reboot_timeout: 120 + when: ( ansible_connection != 'local' and ( ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) and ( reboot_required_file.stat.exists ) ) or ( ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and ( reboot_result.rc == 1 ) ) ) + register: reboot_status + async: 1 + poll: 0 + ignore_errors: True #If unable to reboot (as ansible refuses to do if installing to localhost) we leave the error at the end of the output but don't treat it as a failure. + tags: + - packages + - linux + - linuxdeb + - linuxrpm diff --git a/installer/install_scripts/install_pre.yml b/installer/install_scripts/install_pre.yml new file mode 100644 index 0000000..314aa2c --- /dev/null +++ b/installer/install_scripts/install_pre.yml @@ -0,0 +1,494 @@ +--- +# ansible install playbook that does the prep work for rita V2. +# Version: 202408061413 +# sample runs: +# Optional: Add the following block, without #'s to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports). +# The hosts must each be on their own line. These can be full or short hostnames or a name following "Host" in ~/.ssh/config . +# +# [allritas] +# ro810 +# ub2404 +# +# Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: +# +# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run +# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' + +# Many thanks to but-i-am-dominator for his help with this playbook. + + +- name: "RITA Pre: System prep and checks." + hosts: "{{ install_hosts }}" + become: true + + vars: + ansible_python_interpreter: /bin/python3 # Centos 7 defaults to using python2, so we force python 3. This change does not break any other distros + +#Early tasks needed to support the rest of the install + pre_tasks: +#Known distribution? + - name: "RITA Pre: Checking Linux distribution." + ansible.builtin.fail: + msg: "Distribution name: {{ ansible_distribution }} does not appear to be recognized - please contact ACM" + when: ( ansible_distribution != 'AlmaLinux' and ansible_distribution != 'CentOS' and ansible_distribution != 'Fedora' and ansible_distribution != 'OracleLinux' and ansible_distribution != 'Pop!_OS' and ansible_distribution != 'Rocky' and ansible_distribution != 'Debian' and ansible_distribution != 'Ubuntu' and ansible_distribution != 'Kali' and ansible_distribution != 'Zorin OS' ) + # and ansible_distribution != 'RedHat' + tags: + - linux + + - name: "RITA Pre: Checking Linux distribution version." + ansible.builtin.fail: + msg: "Warning: Linux distribution {{ ansible_distribution }} {{ ansible_distribution_major_version }} may not have been tested - please contact ACM and report whether the install worked or not" + when: ( ( ansible_distribution == 'AlmaLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'CentOS' and (ansible_distribution_major_version != '7' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Fedora' and (ansible_distribution_major_version != '40') ) or ( ansible_distribution == 'OracleLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Pop!_OS' and (ansible_distribution_major_version != '22') ) or ( ansible_distribution == 'Rocky' and (ansible_distribution_major_version != '8' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Debian' and (ansible_distribution_major_version != '12') ) or ( ansible_distribution == 'Kali' and (ansible_distribution_major_version != '2024') ) or ( ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version != '20' and ansible_distribution_major_version != '22' and ansible_distribution_major_version != '24') ) or ( ansible_distribution == 'Zorin OS' and (ansible_distribution_major_version != '16') ) ) + #or ( ansible_distribution != 'RedHat' and (ansible_distribution_major_version == '9') ) + ignore_errors: True #We print a warning but do not abort if this is an unknown combination of distribution and major version. + tags: + - linux + +#CPU Architecture + - name: "RITA Pre: Check system architecture." + ansible.builtin.fail: + msg: "Unsupported CPU architecture: {{ ansible_architecture }}" + when: ( ansible_architecture != "x86_64" ) #and ansible_architecture != "aarch64" ) # "aarch64" for pi. #pi0w is armv6l. i386. amd64? + +#Selinux checks + - name: "RITA Pre: /sys/fs/selinux/enforce exists." + stat: + path: "/sys/fs/selinux/enforce" + check_mode: true + changed_when: false + register: selinuxenforce_check + tags: + - linux + + - name: "RITA Pre: sys filesystem check for selinux." + lineinfile: + path: /sys/fs/selinux/enforce + regexp: '^1' + line: 0 + create: false + unsafe_writes: true #Needed because the original file in the sys filesystem and Ansible's tmp directory are on different filesystems. + state: present + #check_mode: yes + changed_when: false + #register: enforce_check + when: selinuxenforce_check.stat.exists + tags: + - linux + +#Add tools needed by later stages + # Provides "needs-restarting" for ansible's ability to manage rebooting after patching + - name: "RITA Pre: Check for yum-utils before proceeding." + command: rpm -qa | grep yum-utils + check_mode: true + changed_when: false + register: package_check + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + tags: + - packages + - linux + - linuxrpm + + - name: "RITA Pre: Install yum-utils if not found." + package: + name: yum-utils + state: latest + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and '"yum-utils" not in package_check' + tags: + - packages + - linux + - linuxrpm + + # Install aptitude, preferred by ansible for package management on Debian/Ubuntu + - name: "RITA Pre: Install aptitude on debian-based system." + apt: + name: aptitude + state: latest + update_cache: true + cache_valid_time: 3600 + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) #While Kali is based on Debian, it does not include the aptitude package. + tags: + - packages + - linux + - linuxdeb + + + tasks: +# Make sure all rpm packages up to date, add packages + - name: "RITA Pre: Patch and install packages on rpm-based servers." + block: + - name: "RITA Pre: Patch all rpm-based servers." + yum: #We use the "yum" module insteead of dnf to support rpm distros that only have yum + name: "*" + state: latest + skip_broken: yes + update_cache: yes + tags: + - packages + - linux + - linuxrpm + + - name: "RITA Pre: Install rpm packages on rpm-based distributions." + yum: + name: + - nano + - nmap-ncat + - dnf-plugins-core #Provides config-manager binary on Fedora + - wget + - lshw #For user troubleshooting + - net-tools #For user troubleshooting + state: latest + update_cache: true + tags: + - packages + - linux + - linuxrpm + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + + - name: "RITA Pre: Install pip on Centos/Fedora." + yum: + name: + - python3-pip + state: latest + update_cache: true + tags: + - packages + - linux + - linuxrpm + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + # or ansible_distribution == 'OracleLinux' #Note: OracleLinux, and therefore SecurityOnion too, do not include pip3. Disabled. + + - name: "RITA Pre: Patch and install packages on debian-based servers." + block: + - name: "RITA Pre: Patch all debian-based servers." + apt: + name: "*" + state: latest + update_cache: yes + cache_valid_time: 3600 + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Install apt packages on deb-based distributions." + apt: + pkg: + - nano + #Following are to support docker + - apt-transport-https + - ca-certificates + - curl + - python3-pip + - python3-setuptools + - wget + #Following is for user troubleshooting + - net-tools + state: latest + update_cache: true + cache_valid_time: 3600 + tags: + - packages + - linux + - linuxdeb + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) + + + - name: "RITA Pre: Install packages on Debian and Ubuntu." + apt: + pkg: + - ncat #"ncat" is nmap's netcat on Ubuntu and Debian, listd but not available on Kali + - software-properties-common + - virtualenv + - lshw #listed, but somehow not available on Kali + state: latest + update_cache: true + cache_valid_time: 3600 + tags: + - packages + - linux + - linuxdeb + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) + + - name: "RITA Pre: Install packages on Kali." + apt: + pkg: + - netcat-traditional + - python3-virtualenv + state: latest + update_cache: true + cache_valid_time: 3600 + tags: + - packages + - linux + - linuxdeb + when: ( ansible_distribution == 'Kali' ) + + +#Add repositories + - name: "RITA Pre: Add Docker Ubuntu GPG apt key." + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + when: ( ansible_distribution == 'Ubuntu' ) + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker Debian GPG apt key." + apt_key: + url: https://download.docker.com/linux/debian/gpg + state: present + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Zorin OS' ) + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker Repository to Ubuntu or Debian." + apt_repository: + repo: deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable + state: present + when: ( ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' ) + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker Repository to Kali." + apt_repository: + repo: deb https://download.docker.com/linux/debian bookworm stable + state: present + when: ( ansible_distribution == 'Kali' and ansible_distribution_major_version == '2024' ) + #Debian bookworm appears to be the right one to use according to https://www.kali.org/docs/containers/installing-docker-on-kali/ + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker Repository to PopOS." + apt_repository: + repo: deb https://download.docker.com/linux/ubuntu jammy stable + state: present + when: ( ansible_distribution == 'Pop!_OS' and ansible_distribution_major_version == '22' ) + #Ubuntu jammy appears to be the right one to use. + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker Repository to Zorin." + apt_repository: + repo: deb https://download.docker.com/linux/ubuntu focal stable + state: present + when: ( ansible_distribution == 'Zorin OS' and ansible_distribution_major_version == '16' ) + #Ubuntu focal appears to be the right one to use. + tags: + - packages + - linux + - linuxdeb + + - name: "RITA Pre: Add Docker repository to Fedora distributions." + yum_repository: + name: docker-ce + description: Docker package repository + gpgkey: https://download.docker.com/linux/fedora/gpg + baseurl: https://download.docker.com/linux/fedora/$releasever/$basearch/stable/ + state: present + enabled: true + when: ( ansible_distribution == 'Fedora' ) # and ansible_distribution_major_version == '40' ) + tags: + - packages + - linux + - linuxrpm + + - name: "RITA Pre: Add Docker Repository to AlmaLinux/Centos/OracleLinux/Rocky distributions." + #shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + yum_repository: + name: docker-ce + description: Docker package repository + gpgkey: https://download.docker.com/linux/centos/gpg + baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable/ + state: present + enabled: true + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'Rocky' ) + tags: + - packages + - linux + - linuxrpm + + - name: "RITA Pre: Add Docker Repository to RHEL distribution." + yum_repository: + name: docker-ce + description: Docker package repository + gpgkey: https://download.docker.com/linux/rhel/gpg + baseurl: https://download.docker.com/linux/rhel/$releasever/$basearch/stable/ + state: present + enabled: true + when: ( ansible_distribution == 'RedHat' ) + tags: + - packages + - linux + - linuxrpm + + +#Install docker + - name: "RITA Pre: Install docker on debian-based distributions." + block: + - name: "RITA Pre: Uninstall unofficial docker packages on debian-based distributions." + apt: + name: + - docker-client + - docker-client-latest + - docker-common + - docker-compose + - docker-compose-v2 + - docker-doc + - docker-engine + - docker-latest + - docker-latest-logrotate + - docker-logrotate + - docker.io + - podman-docker + state: absent + update_cache: true + cache_valid_time: 3600 + tags: + - docker + - linux + - linuxdeb + + - name: "RITA Pre: Install docker-ce on debian-based distributions." + apt: + name: + - docker-ce + - docker-ce-cli + - docker-compose-plugin + - containerd.io + state: latest + update_cache: true + cache_valid_time: 3600 + tags: + - docker + - linux + - linuxdeb + + - name: "RITA Pre: Install docker modules for Python on deb-based distributions." + apt: + name: + - python3-docker + - python3-requests #We'll have to see if debian/ubuntu can work with the stock (2.28.1 in debian 12.05 / 2.31.0 in ubuntu 24.04) + tags: + - docker + - linux + - linuxdeb + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) + + + - name: "RITA Pre: Install docker on rpm-based distributions." + block: + - name: "RITA Pre: Uninstall unofficial docker packages on rpm-based distributions." + yum: + name: + - docker-client + - docker-client-latest + - docker-common + - docker-compose + - docker-compose-v2 + - docker-doc + - docker-engine-selinux + - docker-engine + - docker-latest + - docker-latest-logrotate + - docker-logrotate + - docker-selinux + - docker.io + - docker + - podman-docker + - podman + - runc + state: absent + update_cache: true + tags: + - docker + - linux + - linuxrpm + + - name: "RITA Pre: Install docker-ce on rpm-based distributions." + yum: + name: + - docker-ce + - docker-ce-cli + - docker-buildx-plugin + - docker-compose-plugin + - containerd.io + state: latest + update_cache: true + tags: + - docker + - linux + - linuxrpm + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + #Reminder that RedHat/RHEL 9 is broken as of 20240618 + + + - name: "RITA Pre: replace python3-requests with a new version installed by pip." + block: + - name: "RITA Pre: Uninstall unofficial docker packages on rpm-based distributions." + yum: + name: + - python3-requests #As of 20240618, issue with requests code: "Error connecting: Error while fetching server API version: Not supported URL scheme http+docker". Installing requests with pip appears to install a newer version that handles the issue. + state: absent + update_cache: true + tags: + - docker + - linux + - linuxrpm + + - name: "RITA Pre: Install docker modules for Python on rpm-based distributions." + pip: + name: + - docker + - requests + tags: + - docker + - linux + - linuxrpm + when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) + #OracleLinux and SecurityOnion don't include pip so we can't do these steps there. + + + - name: "RITA Pre: Start and enable docker in systemd." + systemd: + name: docker + state: started + enabled: yes + when: ( ansible_distribution != 'OracleLinux' ) + tags: + - docker + - linux + - linuxdeb + - linuxrpm + #It appears the "docker modules for python on rpm-based linux" is needed to use the ansible "systemd" module, so we can't use that module on OracleLinux... + + #...so we fall back on starting and enabling it on OracleLinux by hand. + - name: "RITA Pre: Start and enable docker in systemd on OracleLinux." + shell: systemctl enable docker.service ; systemctl start docker.service + when: ( ansible_distribution == 'OracleLinux' ) + tags: + - docker + - linux + - linuxrpm + + - name: "RITA Pre: Transfer docker-compose script to target system for backwards compatibility." + copy: + src: docker-compose + dest: /usr/local/bin/docker-compose + owner: root + group: root + mode: 0755 + tags: + - docker + - linux + - linuxdeb + - linuxrpm diff --git a/installer/install_scripts/install_rita.sh b/installer/install_scripts/install_rita.sh index 847598e..185db32 100755 --- a/installer/install_scripts/install_rita.sh +++ b/installer/install_scripts/install_rita.sh @@ -40,21 +40,22 @@ if [ "$install_target" = "localhost" -o "$install_target" = "127.0.0.1" -o "$ins exit 1 fi status "If asked for a 'BECOME password', that is your non-root sudo password on this machine ." - ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," install_rita.yml if [ "$_INSTALL_ZEEK" = 'true' ]; then - ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," install_zeek.yml + ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," install_pre.yml install_rita.yml install_zeek.yml install_post.yml + else + ansible-playbook --connection=local -K -i "127.0.0.1," -e "install_hosts=127.0.0.1," install_pre.yml install_rita.yml install_post.yml fi else status "Setting up future ssh connections to $install_target . You may be asked to provide your ssh password to $install_target ." #================ ./scripts/sshprep "$install_target" status "If asked for a 'BECOME password', that is your non-root sudo password on $install_target ." - ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," install_rita.yml if [ "$_INSTALL_ZEEK" = 'true' ]; then - ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," install_zeek.yml + ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," install_pre.yml install_rita.yml install_zeek.yml install_post.yml + else + ansible-playbook -K -i "${install_target}," -e "install_hosts=${install_target}," install_pre.yml install_rita.yml install_post.yml fi fi - # ansible-playbook -i ../digitalocean_inventory.py -e "install_hosts=all" install_rita.yml echo \ " diff --git a/installer/install_scripts/install_rita.yml b/installer/install_scripts/install_rita.yml index d6a98e3..fda21c3 100644 --- a/installer/install_scripts/install_rita.yml +++ b/installer/install_scripts/install_rita.yml @@ -1,25 +1,24 @@ --- -#ansible install playbook for rita V2. -#Version: 202406261404 -#sample runs: -# Optional: Add the following block, without #'s to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports). -#The hosts must each be on their own line. These can be full or short hostnames or a name following "Host" in ~/.ssh/config . +# ansible install playbook for rita V2. +# Version: 202408061413 +# sample runs: +# Optional: Add the following block, without #'s to /etc/ansible/hosts (or /opt/local/etc/ansible/hosts if using ansible on mac with mac ports). +# The hosts must each be on their own line. These can be full or short hostnames or a name following "Host" in ~/.ssh/config . # -#[allritas] -#ro810 -#ub2404 +# [allritas] +# ro810 +# ub2404 # -# Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: +# Then run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: # -# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run -# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' +# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run +# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/rita-install.yml | grep -v '^skipping: ' -#Many thanks to but-i-am-dominator for his help with this playbook. +# Many thanks to but-i-am-dominator for his help with this playbook. -- name: "RITA Install: RITA installer and system prep and checks." +- name: "RITA Install: RITA installer." hosts: "{{ install_hosts }}" - #hosts: "{{ install_hosts | default('all') }}" #Not a good idea to fall back on every host in your ansible hosts file. become: true vars: @@ -28,477 +27,9 @@ clickhouse_container_image: clickhouse/clickhouse-server:latest ansible_python_interpreter: /bin/python3 # Centos 7 defaults to using python2, so we force python 3. This change does not break any other distros -#Early tasks needed to support the rest of the install - pre_tasks: -#Known distribution? - - name: "RITA Install: Checking Linux distribution." - ansible.builtin.fail: - msg: "Distribution name: {{ ansible_distribution }} does not appear to be recognized - please contact ACM" - when: ( ansible_distribution != 'AlmaLinux' and ansible_distribution != 'CentOS' and ansible_distribution != 'Fedora' and ansible_distribution != 'OracleLinux' and ansible_distribution != 'Pop!_OS' and ansible_distribution != 'Rocky' and ansible_distribution != 'Debian' and ansible_distribution != 'Ubuntu' and ansible_distribution != 'Kali' and ansible_distribution != 'Zorin OS' ) - # and ansible_distribution != 'RedHat' - tags: - - linux - - - name: "RITA Install: Checking Linux distribution version." - ansible.builtin.fail: - msg: "Warning: Linux distribution {{ ansible_distribution }} {{ ansible_distribution_major_version }} may not have been tested - please contact ACM and report whether the install worked or not" - when: ( ( ansible_distribution == 'AlmaLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'CentOS' and (ansible_distribution_major_version != '7' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Fedora' and (ansible_distribution_major_version != '40') ) or ( ansible_distribution == 'OracleLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Pop!_OS' and (ansible_distribution_major_version != '22') ) or ( ansible_distribution == 'Rocky' and (ansible_distribution_major_version != '8' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Debian' and (ansible_distribution_major_version != '12') ) or ( ansible_distribution == 'Kali' and (ansible_distribution_major_version != '2024') ) or ( ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version != '20' and ansible_distribution_major_version != '22' and ansible_distribution_major_version != '24') ) or ( ansible_distribution == 'Zorin OS' and (ansible_distribution_major_version != '16') ) ) - #or ( ansible_distribution != 'RedHat' and (ansible_distribution_major_version == '9') ) - ignore_errors: True #We print a warning but do not abort if this is an unknown combination of distribution and major version. - tags: - - linux - -#CPU Architecture - - name: "RITA Install: Check system architecture." - ansible.builtin.fail: - msg: "Unsupported CPU architecture: {{ ansible_architecture }}" - when: ( ansible_architecture != "x86_64" ) #and ansible_architecture != "aarch64" ) # "aarch64" for pi. #pi0w is armv6l. i386. amd64? - -#Selinux checks - - name: "RITA Install: /sys/fs/selinux/enforce exists." - stat: - path: "/sys/fs/selinux/enforce" - check_mode: true - changed_when: false - register: selinuxenforce_check - tags: - - linux - - - name: "RITA Install: sys filesystem check for selinux." - lineinfile: - path: /sys/fs/selinux/enforce - regexp: '^1' - line: 0 - create: false - unsafe_writes: true #Needed because the original file in the sys filesystem and Ansible's tmp directory are on different filesystems. - state: present - #check_mode: yes - changed_when: false - #register: enforce_check - when: selinuxenforce_check.stat.exists - tags: - - linux - -#Add tools needed by later stages - # Provides "needs-restarting" for ansible's ability to manage rebooting after patching - - name: "RITA Install: Check for yum-utils before proceeding." - command: rpm -qa | grep yum-utils - check_mode: true - changed_when: false - register: package_check - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - - name: "RITA Install: Install yum-utils if not found." - package: - name: yum-utils - state: latest - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and '"yum-utils" not in package_check' - tags: - - packages - - linux - - linuxrpm - - # Install aptitude, preferred by ansible for package management on Debian/Ubuntu - - name: "RITA Install: Install aptitude on debian-based system." - apt: - name: aptitude - state: latest - update_cache: true - cache_valid_time: 3600 - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) #While Kali is based on Debian, it does not include the aptitude package. - tags: - - packages - - linux - - linuxdeb - +#The install_pre.yml script should already have been run by this point tasks: -# Make sure all rpm packages up to date, add packages - - name: "RITA Install: Patch and install packages on rpm-based servers." - block: - - name: "RITA Install: Patch all rpm-based servers." - yum: #We use the "yum" module insteead of dnf to support rpm distros that only have yum - name: "*" - state: latest - skip_broken: yes - update_cache: yes - tags: - - packages - - linux - - linuxrpm - - - name: "RITA Install: Install rpm packages on rpm-based distributions." - yum: - name: - - nano - - nmap-ncat - - dnf-plugins-core #Provides config-manager binary on Fedora - - wget - - lshw #For user troubleshooting - - net-tools #For user troubleshooting - state: latest - update_cache: true - tags: - - packages - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - - - name: "RITA Install: Install pip on Centos/Fedora." - yum: - name: - - python3-pip - state: latest - update_cache: true - tags: - - packages - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - # or ansible_distribution == 'OracleLinux' #Note: OracleLinux, and therefore SecurityOnion too, do not include pip3. Disabled. - - - name: "RITA Install: Patch and install packages on debian-based servers." - block: - - name: "RITA Install: Patch all debian-based servers." - apt: - name: "*" - state: latest - update_cache: yes - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Install apt packages on deb-based distributions." - apt: - pkg: - - nano - #Following are to support docker - - apt-transport-https - - ca-certificates - - curl - - python3-pip - - python3-setuptools - - wget - #Following is for user troubleshooting - - net-tools - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - - name: "RITA Install: Install packages on Debian and Ubuntu." - apt: - pkg: - - ncat #"ncat" is nmap's netcat on Ubuntu and Debian, listd but not available on Kali - - software-properties-common - - virtualenv - - lshw #listed, but somehow not available on Kali - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - name: "RITA Install: Install packages on Kali." - apt: - pkg: - - netcat-traditional - - python3-virtualenv - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Kali' ) - - - -#Add repositories - - name: "RITA Install: Add Docker Ubuntu GPG apt key." - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - when: ( ansible_distribution == 'Ubuntu' ) - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker Debian GPG apt key." - apt_key: - url: https://download.docker.com/linux/debian/gpg - state: present - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Zorin OS' ) - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker Repository to Ubuntu or Debian." - apt_repository: - repo: deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable - state: present - when: ( ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' ) - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker Repository to Kali." - apt_repository: - repo: deb https://download.docker.com/linux/debian bookworm stable - state: present - when: ( ansible_distribution == 'Kali' and ansible_distribution_major_version == '2024' ) - #Debian bookworm appears to be the right one to use according to https://www.kali.org/docs/containers/installing-docker-on-kali/ - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker Repository to PopOS." - apt_repository: - repo: deb https://download.docker.com/linux/ubuntu jammy stable - state: present - when: ( ansible_distribution == 'Pop!_OS' and ansible_distribution_major_version == '22' ) - #Ubuntu jammy appears to be the right one to use. - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker Repository to Zorin." - apt_repository: - repo: deb https://download.docker.com/linux/ubuntu focal stable - state: present - when: ( ansible_distribution == 'Zorin OS' and ansible_distribution_major_version == '16' ) - #Ubuntu focal appears to be the right one to use. - tags: - - packages - - linux - - linuxdeb - - - name: "RITA Install: Add Docker repository to Fedora distributions." - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/fedora/gpg - baseurl: https://download.docker.com/linux/fedora/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'Fedora' ) # and ansible_distribution_major_version == '40' ) - tags: - - packages - - linux - - linuxrpm - - - name: "RITA Install: Add Docker Repository to AlmaLinux/Centos/OracleLinux/Rocky distributions." - #shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/centos/gpg - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - - name: "RITA Install: Add Docker Repository to RHEL distribution." - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/rhel/gpg - baseurl: https://download.docker.com/linux/rhel/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'RedHat' ) - tags: - - packages - - linux - - linuxrpm - - -#Install docker - - name: "RITA Install: Install docker on debian-based distributions." - block: - - name: "RITA Install: Uninstall unofficial docker packages on debian-based distributions." - apt: - name: - - docker-client - - docker-client-latest - - docker-common - - docker-compose - - docker-compose-v2 - - docker-doc - - docker-engine - - docker-latest - - docker-latest-logrotate - - docker-logrotate - - docker.io - - podman-docker - state: absent - update_cache: true - cache_valid_time: 3600 - tags: - - docker - - linux - - linuxdeb - - - name: "RITA Install: Install docker-ce on debian-based distributions." - apt: - name: - - docker-ce - - docker-ce-cli - - docker-compose-plugin - - containerd.io - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - docker - - linux - - linuxdeb - - - name: "RITA Install: Install docker modules for Python on deb-based distributions." - apt: - name: - - python3-docker - - python3-requests #We'll have to see if debian/ubuntu can work with the stock (2.28.1 in debian 12.05 / 2.31.0 in ubuntu 24.04) - tags: - - docker - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - - name: "RITA Install: Install docker on rpm-based distributions." - block: - - name: "RITA Install: Uninstall unofficial docker packages on rpm-based distributions." - yum: - name: - - docker-client - - docker-client-latest - - docker-common - - docker-compose - - docker-compose-v2 - - docker-doc - - docker-engine-selinux - - docker-engine - - docker-latest - - docker-latest-logrotate - - docker-logrotate - - docker-selinux - - docker.io - - docker - - podman-docker - - podman - - runc - state: absent - update_cache: true - tags: - - docker - - linux - - linuxrpm - - - name: "RITA Install: Install docker-ce on rpm-based distributions." - yum: - name: - - docker-ce - - docker-ce-cli - - docker-buildx-plugin - - docker-compose-plugin - - containerd.io - state: latest - update_cache: true - tags: - - docker - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - #Reminder that RedHat/RHEL 9 is broken as of 20240618 - - - - name: "RITA Install: replace python3-requests with a new version installed by pip." - block: - - name: "RITA Install: Uninstall unofficial docker packages on rpm-based distributions." - yum: - name: - - python3-requests #As of 20240618, issue with requests code: "Error connecting: Error while fetching server API version: Not supported URL scheme http+docker". Installing requests with pip appears to install a newer version that handles the issue. - state: absent - update_cache: true - tags: - - docker - - linux - - linuxrpm - - - name: "RITA Install: Install docker modules for Python on rpm-based distributions." - pip: - name: - - docker - - requests - tags: - - docker - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - #OracleLinux and SecurityOnion don't include pip so we can't do these steps there. - - - - name: "RITA Install: Start and enable docker in systemd." - systemd: - name: docker - state: started - enabled: yes - when: ( ansible_distribution != 'OracleLinux' ) - tags: - - docker - - linux - - linuxdeb - - linuxrpm - #It appears the "docker modules for python on rpm-based linux" is needed to use the ansible "systemd" module, so we can't use that module on OracleLinux... - - #...so we fall back on starting and enabling it on OracleLinux by hand. - - name: "RITA Install: Start and enable docker in systemd on OracleLinux." - shell: systemctl enable docker.service ; systemctl start docker.service - when: ( ansible_distribution == 'OracleLinux' ) - tags: - - docker - - linux - - linuxrpm - - - name: "RITA Install: Transfer docker-compose script to target system for backwards compatibility." - copy: - src: docker-compose - dest: /usr/local/bin/docker-compose - owner: root - group: root - mode: 0755 - tags: - - docker - - rita - - linux - - linuxdeb - - linuxrpm - #Make directories - name: "RITA Install: Create configuration directories." ansible.builtin.file: @@ -591,38 +122,5 @@ - linuxdeb - linuxrpm -#Late tasks, including rebooting - - name: "RITA Install: Check if reboot required on rpm-based systems." - command: needs-restarting -r - register: reboot_result - ignore_errors: true - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - - name: "RITA Install: Check if reboot required on Debian/Ubuntu-based systems." - register: reboot_required_file - stat: - path: /var/run/reboot-required - get_checksum: no - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - tags: - - packages - - linux - - linuxdeb - - name: "RITA Install: Rebooting system if needed." - reboot: - reboot_timeout: 120 - when: ( ansible_connection != 'local' and ( ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) and ( reboot_required_file.stat.exists ) ) or ( ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and ( reboot_result.rc == 1 ) ) ) - register: reboot_status - async: 1 - poll: 0 - ignore_errors: True #If unable to reboot (as ansible refuses to do if installing to localhost) we leave the error at the end of the output but don't treat it as a failure. - tags: - - packages - - linux - - linuxdeb - - linuxrpm +#The install_post.yml script should be run next diff --git a/installer/install_scripts/install_zeek.yml b/installer/install_scripts/install_zeek.yml index c017b35..52f6bb7 100644 --- a/installer/install_scripts/install_zeek.yml +++ b/installer/install_scripts/install_zeek.yml @@ -1,18 +1,17 @@ --- -#ansible install playbook for docker-zeek. -#Version: 202407220000 -#sample runs: -# Run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: +# ansible install playbook for docker-zeek. +# Version: 202408061413 +# sample runs: +# Run this, with a comma separated list of hostnames from the above file with a comma at the end of the list: # -# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/install_zeek.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run -# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/install_zeek.yml | grep -v '^skipping: ' +# ansible-playbook -C -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/install_zeek.yml | grep -v '^skipping: ' #-C (no changes) means do a dry run +# ansible-playbook -K -i "ro810,ub2404," -e "install_hosts=ro810,ub2404," ~/.ansible/playbooks/install_zeek.yml | grep -v '^skipping: ' -#Many thanks to but-i-am-dominator for his help with this playbook. +# Many thanks to but-i-am-dominator for his help with this playbook. -- name: "Zeek Install: Zeek installer and system prep and checks." +- name: "Zeek Install: Zeek installer." hosts: "{{ install_hosts }}" - #hosts: "{{ install_hosts | default('all') }}" #Not a good idea to fall back on every host in your ansible hosts file. become: true vars: @@ -21,477 +20,9 @@ clickhouse_container_image: clickhouse/clickhouse-server:latest ansible_python_interpreter: /bin/python3 # Centos 7 defaults to using python2, so we force python 3. This change does not break any other distros -#Early tasks needed to support the rest of the install - pre_tasks: -#Known distribution? - - name: "Zeek Install: Checking Linux distribution." - ansible.builtin.fail: - msg: "Distribution name: {{ ansible_distribution }} does not appear to be recognized - please contact ACM" - when: ( ansible_distribution != 'AlmaLinux' and ansible_distribution != 'CentOS' and ansible_distribution != 'Fedora' and ansible_distribution != 'OracleLinux' and ansible_distribution != 'Pop!_OS' and ansible_distribution != 'Rocky' and ansible_distribution != 'Debian' and ansible_distribution != 'Ubuntu' and ansible_distribution != 'Kali' and ansible_distribution != 'Zorin OS' ) - # and ansible_distribution != 'RedHat' - tags: - - linux - - - name: "Zeek Install: Checking Linux distribution version." - ansible.builtin.fail: - msg: "Warning: Linux distribution {{ ansible_distribution }} {{ ansible_distribution_major_version }} may not have been tested - please contact ACM and report whether the install worked or not" - when: ( ( ansible_distribution == 'AlmaLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'CentOS' and (ansible_distribution_major_version != '7' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Fedora' and (ansible_distribution_major_version != '40') ) or ( ansible_distribution == 'OracleLinux' and (ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Pop!_OS' and (ansible_distribution_major_version != '22') ) or ( ansible_distribution == 'Rocky' and (ansible_distribution_major_version != '8' and ansible_distribution_major_version != '9') ) or ( ansible_distribution == 'Debian' and (ansible_distribution_major_version != '12') ) or ( ansible_distribution == 'Kali' and (ansible_distribution_major_version != '2024') ) or ( ansible_distribution == 'Ubuntu' and (ansible_distribution_major_version != '20' and ansible_distribution_major_version != '22' and ansible_distribution_major_version != '24') ) or ( ansible_distribution == 'Zorin OS' and (ansible_distribution_major_version != '16') ) ) - #or ( ansible_distribution != 'RedHat' and (ansible_distribution_major_version == '9') ) - ignore_errors: True #We print a warning but do not abort if this is an unknown combination of distribution and major version. - tags: - - linux - -#CPU Architecture - - name: "Zeek Install: Check system architecture." - ansible.builtin.fail: - msg: "Unsupported CPU architecture: {{ ansible_architecture }}" - when: ( ansible_architecture != "x86_64" ) #and ansible_architecture != "aarch64" ) # "aarch64" for pi. #pi0w is armv6l. i386. amd64? - -#Selinux checks - - name: "Zeek Install: /sys/fs/selinux/enforce exists." - stat: - path: "/sys/fs/selinux/enforce" - check_mode: true - changed_when: false - register: selinuxenforce_check - tags: - - linux - - - name: "Zeek Install: sys filesystem check for selinux." - lineinfile: - path: /sys/fs/selinux/enforce - regexp: '^1' - line: 0 - create: false - unsafe_writes: true #Needed because the original file in the sys filesystem and Ansible's tmp directory are on different filesystems. - state: present - #check_mode: yes - changed_when: false - #register: enforce_check - when: selinuxenforce_check.stat.exists - tags: - - linux - -#Add tools needed by later stages - # Provides "needs-restarting" for ansible's ability to manage rebooting after patching - - name: "Zeek Install: Check for yum-utils before proceeding." - command: rpm -qa | grep yum-utils - check_mode: true - changed_when: false - register: package_check - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - - name: "Zeek Install: Install yum-utils if not found." - package: - name: yum-utils - state: latest - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and '"yum-utils" not in package_check' - tags: - - packages - - linux - - linuxrpm - - # Install aptitude, preferred by ansible for package management on Debian/Ubuntu - - name: "Zeek Install: Install aptitude on debian-based system." - apt: - name: aptitude - state: latest - update_cache: true - cache_valid_time: 3600 - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) #While Kali is based on Debian, it does not include the aptitude package. - tags: - - packages - - linux - - linuxdeb - +#The install_pre.yml script should already have been run by this point tasks: -# Make sure all rpm packages up to date, add packages - - name: "Zeek Install: Patch and install packages on rpm-based servers." - block: - - name: "Zeek Install: Patch all rpm-based servers." - yum: #We use the "yum" module insteead of dnf to support rpm distros that only have yum - name: "*" - state: latest - skip_broken: yes - update_cache: yes - tags: - - packages - - linux - - linuxrpm - - - name: "Zeek Install: Install rpm packages on rpm-based distributions." - yum: - name: - - nano - - nmap-ncat - - dnf-plugins-core #Provides config-manager binary on Fedora - - wget - - lshw #For user troubleshooting - - net-tools #For user troubleshooting - state: latest - update_cache: true - tags: - - packages - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - - - name: "Zeek Install: Install pip on Centos/Fedora." - yum: - name: - - python3-pip - state: latest - update_cache: true - tags: - - packages - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - # or ansible_distribution == 'OracleLinux' #Note: OracleLinux, and therefore SecurityOnion too, do not include pip3. Disabled. - - - name: "Zeek Install: Patch and install packages on debian-based servers." - block: - - name: "Zeek Install: Patch all debian-based servers." - apt: - name: "*" - state: latest - update_cache: yes - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Install apt packages on deb-based distributions." - apt: - pkg: - - nano - #Following are to support docker - - apt-transport-https - - ca-certificates - - curl - - python3-pip - - python3-setuptools - - wget - #Following is for user troubleshooting - - net-tools - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - - name: "Zeek Install: Install packages on Debian and Ubuntu." - apt: - pkg: - - ncat #"ncat" is nmap's netcat on Ubuntu and Debian, listd but not available on Kali - - software-properties-common - - virtualenv - - lshw #listed, but somehow not available on Kali - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - name: "Zeek Install: Install packages on Kali." - apt: - pkg: - - netcat-traditional - - python3-virtualenv - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - packages - - linux - - linuxdeb - when: ( ansible_distribution == 'Kali' ) - - - -#Add repositories - - name: "Zeek Install: Add Docker Ubuntu GPG apt key." - apt_key: - url: https://download.docker.com/linux/ubuntu/gpg - state: present - when: ( ansible_distribution == 'Ubuntu' ) - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker Debian GPG apt key." - apt_key: - url: https://download.docker.com/linux/debian/gpg - state: present - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Zorin OS' ) - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker Repository to Ubuntu or Debian." - apt_repository: - repo: deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable - state: present - when: ( ansible_distribution == 'Ubuntu' or ansible_distribution == 'Debian' ) - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker Repository to Kali." - apt_repository: - repo: deb https://download.docker.com/linux/debian bookworm stable - state: present - when: ( ansible_distribution == 'Kali' and ansible_distribution_major_version == '2024' ) - #Debian bookworm appears to be the right one to use according to https://www.kali.org/docs/containers/installing-docker-on-kali/ - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker Repository to PopOS." - apt_repository: - repo: deb https://download.docker.com/linux/ubuntu jammy stable - state: present - when: ( ansible_distribution == 'Pop!_OS' and ansible_distribution_major_version == '22' ) - #Ubuntu jammy appears to be the right one to use. - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker Repository to Zorin." - apt_repository: - repo: deb https://download.docker.com/linux/ubuntu focal stable - state: present - when: ( ansible_distribution == 'Zorin OS' and ansible_distribution_major_version == '16' ) - #Ubuntu focal appears to be the right one to use. - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Add Docker repository to Fedora distributions." - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/fedora/gpg - baseurl: https://download.docker.com/linux/fedora/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'Fedora' ) # and ansible_distribution_major_version == '40' ) - tags: - - packages - - linux - - linuxrpm - - - name: "Zeek Install: Add Docker Repository to AlmaLinux/Centos/OracleLinux/Rocky distributions." - #shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/centos/gpg - baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - - name: "Zeek Install: Add Docker Repository to RHEL distribution." - yum_repository: - name: docker-ce - description: Docker package repository - gpgkey: https://download.docker.com/linux/rhel/gpg - baseurl: https://download.docker.com/linux/rhel/$releasever/$basearch/stable/ - state: present - enabled: true - when: ( ansible_distribution == 'RedHat' ) - tags: - - packages - - linux - - linuxrpm - - -#Install docker - - name: "Zeek Install: Install docker on debian-based distributions." - block: - - name: "Zeek Install: Uninstall unofficial docker packages on debian-based distributions." - apt: - name: - - docker-client - - docker-client-latest - - docker-common - - docker-compose - - docker-compose-v2 - - docker-doc - - docker-engine - - docker-latest - - docker-latest-logrotate - - docker-logrotate - - docker.io - - podman-docker - state: absent - update_cache: true - cache_valid_time: 3600 - tags: - - docker - - linux - - linuxdeb - - - name: "Zeek Install: Install docker-ce on debian-based distributions." - apt: - name: - - docker-ce - - docker-ce-cli - - docker-compose-plugin - - containerd.io - state: latest - update_cache: true - cache_valid_time: 3600 - tags: - - docker - - linux - - linuxdeb - - - name: "Zeek Install: Install docker modules for Python on deb-based distributions." - apt: - name: - - python3-docker - - python3-requests #We'll have to see if debian/ubuntu can work with the stock (2.28.1 in debian 12.05 / 2.31.0 in ubuntu 24.04) - tags: - - docker - - linux - - linuxdeb - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - - - - name: "Zeek Install: Install docker on rpm-based distributions." - block: - - name: "Zeek Install: Uninstall unofficial docker packages on rpm-based distributions." - yum: - name: - - docker-client - - docker-client-latest - - docker-common - - docker-compose - - docker-compose-v2 - - docker-doc - - docker-engine-selinux - - docker-engine - - docker-latest - - docker-latest-logrotate - - docker-logrotate - - docker-selinux - - docker.io - - docker - - podman-docker - - podman - - runc - state: absent - update_cache: true - tags: - - docker - - linux - - linuxrpm - - - name: "Zeek Install: Install docker-ce on rpm-based distributions." - yum: - name: - - docker-ce - - docker-ce-cli - - docker-buildx-plugin - - docker-compose-plugin - - containerd.io - state: latest - update_cache: true - tags: - - docker - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - #Reminder that RedHat/RHEL 9 is broken as of 20240618 - - - - name: "Zeek Install: replace python3-requests with a new version installed by pip." - block: - - name: "Zeek Install: Uninstall unofficial docker packages on rpm-based distributions." - yum: - name: - - python3-requests #As of 20240618, issue with requests code: "Error connecting: Error while fetching server API version: Not supported URL scheme http+docker". Installing requests with pip appears to install a newer version that handles the issue. - state: absent - update_cache: true - tags: - - docker - - linux - - linuxrpm - - - name: "Zeek Install: Install docker modules for Python on rpm-based distributions." - pip: - name: - - docker - - requests - tags: - - docker - - linux - - linuxrpm - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - #OracleLinux and SecurityOnion don't include pip so we can't do these steps there. - - - - name: "Zeek Install: Start and enable docker in systemd." - systemd: - name: docker - state: started - enabled: yes - when: ( ansible_distribution != 'OracleLinux' ) - tags: - - docker - - linux - - linuxdeb - - linuxrpm - #It appears the "docker modules for python on rpm-based linux" is needed to use the ansible "systemd" module, so we can't use that module on OracleLinux... - - #...so we fall back on starting and enabling it on OracleLinux by hand. - - name: "Zeek Install: Start and enable docker in systemd on OracleLinux." - shell: systemctl enable docker.service ; systemctl start docker.service - when: ( ansible_distribution == 'OracleLinux' ) - tags: - - docker - - linux - - linuxrpm - - - name: "Zeek Install: Transfer docker-compose script to target system for backwards compatibility." - copy: - src: docker-compose - dest: /usr/local/bin/docker-compose - owner: root - group: root - mode: 0755 - tags: - - docker - - zeek - - linux - - linuxdeb - - linuxrpm - #Make directories - name: "Zeek Install: Create zeek directories." ansible.builtin.file: @@ -596,38 +127,5 @@ # - linuxdeb # - linuxrpm -#Late tasks, including rebooting - - name: "Zeek Install: Check if reboot required on rpm-based systems." - command: needs-restarting -r - register: reboot_result - ignore_errors: true - when: ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) - tags: - - packages - - linux - - linuxrpm - - name: "Zeek Install: Check if reboot required on Debian/Ubuntu-based systems." - register: reboot_required_file - stat: - path: /var/run/reboot-required - get_checksum: no - when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) - tags: - - packages - - linux - - linuxdeb - - - name: "Zeek Install: Rebooting system if needed." - reboot: - reboot_timeout: 120 - when: ( ansible_connection != 'local' and ( ( ansible_distribution == 'Debian' or ansible_distribution == 'Kali' or ansible_distribution == 'Pop!_OS' or ansible_distribution == 'Ubuntu' or ansible_distribution == 'Zorin OS' ) and ( reboot_required_file.stat.exists ) ) or ( ( ansible_distribution == 'AlmaLinux' or ansible_distribution == 'CentOS' or ansible_distribution == 'Fedora' or ansible_distribution == 'OracleLinux' or ansible_distribution == 'RedHat' or ansible_distribution == 'Rocky' ) and ( reboot_result.rc == 1 ) ) ) - register: reboot_status - async: 1 - poll: 0 - ignore_errors: True #If unable to reboot (as ansible refuses to do if installing to localhost) we leave the error at the end of the output but don't treat it as a failure. - tags: - - packages - - linux - - linuxdeb - - linuxrpm +#The install_post.yml script should be run next