diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4405b6caa5d08..682e5dfb750db 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -11,7 +11,7 @@ # experience as rich as possible. Perhaps later down the line it might be worth # rolling our own # -FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bullseye +FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bookworm # Make sure all exit codes on pipes cause failures SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -31,9 +31,9 @@ CMD ["sleep", "infinity"] RUN apt-get update \ && apt-get -y install --no-install-recommends \ # Add in useful db debugging tools - "postgresql-client=13+*" \ + "postgresql-client=15+*" \ # needed for posthog to run - netcat brotli curl \ + netcat-openbsd brotli curl \ && rm -rf /var/lib/apt/lists/* RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ diff --git a/.devcontainer/library-scripts/docker-in-docker-debian.sh b/.devcontainer/library-scripts/docker-in-docker-debian.sh index 476e1ef3d38b6..893293f973dae 100755 --- a/.devcontainer/library-scripts/docker-in-docker-debian.sh +++ b/.devcontainer/library-scripts/docker-in-docker-debian.sh @@ -5,22 +5,29 @@ #------------------------------------------------------------------------------------------------------------- # # Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker-in-docker.md -# Maintainer: The VS Code and Codespaces Teams -# -# Syntax: ./docker-in-docker-debian.sh [enable non-root docker access flag] [non-root user] [use moby] [Engine/CLI Version] [Major version for docker-compose] - -ENABLE_NONROOT_DOCKER=${1:-"true"} -USERNAME=${2:-"automatic"} -USE_MOBY=${3:-"true"} -DOCKER_VERSION=${4:-"latest"} # The Docker/Moby Engine + CLI should match in version -DOCKER_DASH_COMPOSE_VERSION=${5:-"v1"} # v1 or v2 +# Maintainer: The Dev Container spec maintainers + + +DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version +USE_MOBY="${MOBY:-"true"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"latest"}" #latest, v2 or none +AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" +DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" +USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" +INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" +INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" +DISABLE_IP6_TABLES="${DISABLEIP6TABLES:-false}" # Default: Exit on any failure. set -e +# Clean up +rm -rf /var/lib/apt/lists/* + # Setup STDERR. err() { echo "(!) $*" >&2 @@ -40,7 +47,7 @@ fi if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then USERNAME="" POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do if id -u ${CURRENT_USER} > /dev/null 2>&1; then USERNAME=${CURRENT_USER} break @@ -53,36 +60,18 @@ elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then USERNAME=root fi -# Get central common setting -get_common_setting() { - if [ "${common_settings_file_loaded}" != "true" ]; then - curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping." - common_settings_file_loaded=true - fi - if [ -f "/tmp/vsdc-settings.env" ]; then - local multi_line="" - if [ "$2" = "true" ]; then multi_line="-z"; fi - local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')" - if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi - fi - echo "$1=${!1}" -} - -# Function to run apt-get if needed -apt_get_update_if_needed() +apt_get_update() { - if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then echo "Running apt-get update..." - apt-get update - else - echo "Skipping apt-get update." + apt-get update -y fi } # Checks if packages are installed and installs them if not check_packages() { if ! dpkg -s "$@" > /dev/null 2>&1; then - apt_get_update_if_needed + apt_get_update apt-get -y install --no-install-recommends "$@" fi } @@ -95,7 +84,7 @@ find_version_from_git_tags() { local repository=$2 local prefix=${3:-"tags/v"} local separator=${4:-"."} - local last_part_optional=${5:-"false"} + local last_part_optional=${5:-"false"} if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator=${separator//./\\.} local last_part @@ -121,6 +110,75 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + ########################################### # Start docker-in-docker installation ########################################### @@ -134,10 +192,8 @@ export DEBIAN_FRONTEND=noninteractive # Fetch host/container arch. architecture="$(dpkg --print-architecture)" -# Check if distro is suppported +# Check if distro is supported if [ "${USE_MOBY}" = "true" ]; then - # 'get_common_setting' allows attribute to be updated remotely - get_common_setting DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES if [[ "${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS distribution" err "Support distributions include: ${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" @@ -145,7 +201,6 @@ if [ "${USE_MOBY}" = "true" ]; then fi echo "Distro codename '${VERSION_CODENAME}' matched filter '${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}'" else - get_common_setting DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES if [[ "${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, please choose a compatible OS distribution" err "Support distributions include: ${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" @@ -155,10 +210,9 @@ else fi # Install dependencies -check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr +check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr wget jq if ! type git > /dev/null 2>&1; then - apt_get_update_if_needed - apt-get -y install git + check_packages git fi # Swap to legacy iptables for compatibility @@ -177,7 +231,6 @@ if [ "${USE_MOBY}" = "true" ]; then cli_package_name="moby-cli" # Import key safely and import Microsoft apt repo - get_common_setting MICROSOFT_GPG_KEYS_URI curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list else @@ -217,6 +270,27 @@ else echo "cli_version_suffix ${cli_version_suffix}" fi +# Version matching for moby-buildx +if [ "${USE_MOBY}" = "true" ]; then + if [ "${MOBY_BUILDX_VERSION}" = "latest" ]; then + # Empty, meaning grab whatever "latest" is in apt repo + buildx_version_suffix="" + else + buildx_version_dot_escaped="${MOBY_BUILDX_VERSION//./\\.}" + buildx_version_dot_plus_escaped="${buildx_version_dot_escaped//+/\\+}" + buildx_version_regex="^(.+:)?${buildx_version_dot_plus_escaped}([\\.\\+ ~:-]|$)" + set +e + buildx_version_suffix="=$(apt-cache madison moby-buildx | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${buildx_version_regex}")" + set -e + if [ -z "${buildx_version_suffix}" ] || [ "${buildx_version_suffix}" = "=" ]; then + err "No full or partial moby-buildx version match found for \"${MOBY_BUILDX_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:" + apt-cache madison moby-buildx | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+' + exit 1 + fi + echo "buildx_version_suffix ${buildx_version_suffix}" + fi +fi + # Install Docker / Moby CLI if not already installed if type docker > /dev/null 2>&1 && type dockerd > /dev/null 2>&1; then echo "Docker / Moby CLI and Engine already installed." @@ -224,97 +298,196 @@ else if [ "${USE_MOBY}" = "true" ]; then # Install engine set +e # Handle error gracefully - apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx moby-engine${engine_version_suffix} - if [ $? -ne 0 ]; then - err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." - exit 1 - fi - set -e + apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx${buildx_version_suffix} moby-engine${engine_version_suffix} + exit_code=$? + set -e + + if [ ${exit_code} -ne 0 ]; then + err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." + exit 1 + fi # Install compose apt-get -y install --no-install-recommends moby-compose || err "Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." else apt-get -y install --no-install-recommends docker-ce-cli${cli_version_suffix} docker-ce${engine_version_suffix} + # Install compose + apt-get -y install --no-install-recommends docker-compose-plugin || echo "(*) Package docker-compose-plugin (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." fi fi echo "Finished installing docker / moby!" -# Install Docker Compose if not already installed and is on a supported architecture -if type docker-compose > /dev/null 2>&1; then - echo "Docker Compose v1 already installed." -else - target_compose_arch="${architecture}" - if [ "${target_compose_arch}" = "amd64" ]; then - target_compose_arch="x86_64" - fi - if [ "${target_compose_arch}" != "x86_64" ]; then - # Use pip to get a version that runs on this architecture - if ! dpkg -s python3-minimal python3-pip libffi-dev python3-venv > /dev/null 2>&1; then - apt_get_update_if_needed - apt-get -y install python3-minimal python3-pip libffi-dev python3-venv - fi - export PIPX_HOME=/usr/local/pipx - mkdir -p ${PIPX_HOME} - export PIPX_BIN_DIR=/usr/local/bin - export PYTHONUSERBASE=/tmp/pip-tmp - export PIP_CACHE_DIR=/tmp/pip-tmp/cache - pipx_bin=pipx - if ! type pipx > /dev/null 2>&1; then - pip3 install --disable-pip-version-check --no-cache-dir --user pipx - pipx_bin=/tmp/pip-tmp/bin/pipx +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +# fallback for docker-compose +fallback_compose(){ + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "${url}" "${repo_url}" compose_version + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + +# If 'docker-compose' command is to be included +if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then + case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 + esac + + docker_compose_path="/usr/local/bin/docker-compose" + if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then + err "The final Compose V1 release, version 1.29.2, was May 10, 2021. These packages haven't received any security updates since then. Use at your own risk." + INSTALL_DOCKER_COMPOSE_SWITCH="false" + + if [ "${target_compose_arch}" = "x86_64" ]; then + echo "(*) Installing docker compose v1..." + curl -fsSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64" -o ${docker_compose_path} + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + elif [ "${VERSION_CODENAME}" = "bookworm" ]; then + err "Docker compose v1 is unavailable for 'bookworm' on Arm64. Kindly switch to use v2" + exit 1 + else + # Use pip to get a version that runs on this architecture + check_packages python3-minimal python3-pip libffi-dev python3-venv + echo "(*) Installing docker compose v1 via pip..." + export PYTHONUSERBASE=/usr/local + pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation fi - ${pipx_bin} install --pip-args '--no-cache-dir --force-reinstall' docker-compose - rm -rf /tmp/pip-tmp else - compose_v1_version="1" - find_version_from_git_tags compose_v1_version "https://github.com/docker/compose" "tags/" - echo "(*) Installing docker-compose ${compose_v1_version}..." - curl -fsSL "https://github.com/docker/compose/releases/download/${compose_v1_version}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} + docker_compose_url="https://github.com/docker/compose" + find_version_from_git_tags compose_version "$docker_compose_url" "tags/v" + echo "(*) Installing docker-compose ${compose_version}..." + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || { + if [[ $DOCKER_DASH_COMPOSE_VERSION == "latest" ]]; then + fallback_compose "$docker_compose_url" + else + echo -e "Error: Failed to install docker-compose v${compose_version}" + fi + } + + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + + mkdir -p ${cli_plugins_dir} + cp ${docker_compose_path} ${cli_plugins_dir} fi fi -# Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation -current_v1_compose_path="$(which docker-compose)" -target_v1_compose_path="$(dirname "${current_v1_compose_path}")/docker-compose-v1" -if ! type compose-switch > /dev/null 2>&1; then - echo "(*) Installing compose-switch..." - compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" +# fallback method for compose-switch +fallback_compose-switch() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "$url" "$repo_url" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch - chmod +x /usr/local/bin/compose-switch - # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 +} - # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) - mv "${current_v1_compose_path}" "${target_v1_compose_path}" - update-alternatives --install /usr/local/bin/docker-compose docker-compose /usr/local/bin/compose-switch 99 - update-alternatives --install /usr/local/bin/docker-compose docker-compose "${target_v1_compose_path}" 1 -fi -if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then - update-alternatives --set docker-compose "${target_v1_compose_path}" -else - update-alternatives --set docker-compose /usr/local/bin/compose-switch +# Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation +if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then + if type docker-compose > /dev/null 2>&1; then + echo "(*) Installing compose-switch..." + current_compose_path="$(which docker-compose)" + target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" + compose_switch_version="latest" + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "$compose_switch_url" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || fallback_compose-switch "$compose_switch_url" + chmod +x /usr/local/bin/compose-switch + # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 + # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) + mv "${current_compose_path}" "${target_compose_path}" + update-alternatives --install ${docker_compose_path} docker-compose /usr/local/bin/compose-switch 99 + update-alternatives --install ${docker_compose_path} docker-compose "${target_compose_path}" 1 + else + err "Skipping installation of compose-switch as docker compose is unavailable..." + fi fi # If init file already exists, exit if [ -f "/usr/local/share/docker-init.sh" ]; then echo "/usr/local/share/docker-init.sh already exists, so exiting." + # Clean up + rm -rf /var/lib/apt/lists/* exit 0 fi -echo "docker-init doesnt exist, adding..." +echo "docker-init doesn't exist, adding..." -# Add user to the docker group -if [ "${ENABLE_NONROOT_DOCKER}" = "true" ]; then - if ! getent group docker > /dev/null 2>&1; then - groupadd docker - fi +if ! cat /etc/group | grep -e "^docker:" > /dev/null 2>&1; then + groupadd -r docker +fi - usermod -aG docker ${USERNAME} +usermod -aG docker ${USERNAME} + +# fallback for docker/buildx +fallback_buildx() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "$url" "$repo_url" buildx_version + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} +} + +if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then + buildx_version="latest" + docker_buildx_url="https://github.com/docker/buildx" + find_version_from_git_tags buildx_version "$docker_buildx_url" "refs/tags/v" + echo "(*) Installing buildx ${buildx_version}..." + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + + cd /tmp + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || fallback_buildx "$docker_buildx_url" + + docker_home="/usr/libexec/docker" + cli_plugins_dir="${docker_home}/cli-plugins" + + mkdir -p ${cli_plugins_dir} + mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx + chmod +x ${cli_plugins_dir}/docker-buildx + + chown -R "${USERNAME}:docker" "${docker_home}" + chmod -R g+r+w "${docker_home}" + find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s +fi + +DOCKER_DEFAULT_IP6_TABLES="" +if [ "$DISABLE_IP6_TABLES" == true ]; then + requested_version="" + # checking whether the version requested either is in semver format or just a number denoting the major version + # and, extracting the major version number out of the two scenarios + semver_regex="^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$" + if echo "$DOCKER_VERSION" | grep -Eq $semver_regex; then + requested_version=$(echo $DOCKER_VERSION | cut -d. -f1) + elif echo "$DOCKER_VERSION" | grep -Eq "^[1-9][0-9]*$"; then + requested_version=$DOCKER_VERSION + fi + if [ "$DOCKER_VERSION" = "latest" ] || [[ -n "$requested_version" && "$requested_version" -ge 27 ]] ; then + DOCKER_DEFAULT_IP6_TABLES="--ip6tables=false" + echo "(!) As requested, passing '${DOCKER_DEFAULT_IP6_TABLES}'" + fi fi tee /usr/local/share/docker-init.sh > /dev/null \ -<< 'EOF' +<< EOF #!/bin/sh #------------------------------------------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. @@ -323,13 +496,19 @@ tee /usr/local/share/docker-init.sh > /dev/null \ set -e -dockerd_start="$(cat << 'INNEREOF' +AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} +DOCKER_DEFAULT_ADDRESS_POOL=${DOCKER_DEFAULT_ADDRESS_POOL} +DOCKER_DEFAULT_IP6_TABLES=${DOCKER_DEFAULT_IP6_TABLES} +EOF + +tee -a /usr/local/share/docker-init.sh > /dev/null \ +<< 'EOF' +dockerd_start="AZURE_DNS_AUTO_DETECTION=${AZURE_DNS_AUTO_DETECTION} DOCKER_DEFAULT_ADDRESS_POOL=${DOCKER_DEFAULT_ADDRESS_POOL} DOCKER_DEFAULT_IP6_TABLES=${DOCKER_DEFAULT_IP6_TABLES} $(cat << 'INNEREOF' # explicitly remove dockerd and containerd PID file to ensure that it can start properly if it was stopped uncleanly - # ie: docker kill find /run /var/run -iname 'docker*.pid' -delete || : find /run /var/run -iname 'container*.pid' -delete || : - ## Dind wrapper script from docker team, adapted to a function + # -- Start: dind wrapper script -- # Maintained: https://github.com/moby/moby/blob/master/hack/dind export container=docker @@ -346,45 +525,109 @@ dockerd_start="$(cat << 'INNEREOF' mount -t tmpfs none /tmp fi - # cgroup v2: enable nesting - if [ -f /sys/fs/cgroup/cgroup.controllers ]; then - # move the processes from the root group to the /init group, - # otherwise writing subtree_control fails with EBUSY. - # An error during moving non-existent process (i.e., "cat") is ignored. - mkdir -p /sys/fs/cgroup/init - xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : - # enable controllers - sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ - > /sys/fs/cgroup/cgroup.subtree_control - fi - ## Dind wrapper over. + set_cgroup_nesting() + { + # cgroup v2: enable nesting + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + # move the processes from the root group to the /init group, + # otherwise writing subtree_control fails with EBUSY. + # An error during moving non-existent process (i.e., "cat") is ignored. + mkdir -p /sys/fs/cgroup/init + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + # enable controllers + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control + fi + } + + # Set cgroup nesting, retrying if necessary + retry_cgroup_nesting=0 + + until [ "${retry_cgroup_nesting}" -eq "5" ]; + do + set +e + set_cgroup_nesting + + if [ $? -ne 0 ]; then + echo "(*) cgroup v2: Failed to enable nesting, retrying..." + else + break + fi + + retry_cgroup_nesting=`expr $retry_cgroup_nesting + 1` + set -e + done + + # -- End: dind wrapper script -- # Handle DNS set +e - cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' - if [ $? -eq 0 ] + cat /etc/resolv.conf | grep -i 'internal.cloudapp.net' > /dev/null 2>&1 + if [ $? -eq 0 ] && [ "${AZURE_DNS_AUTO_DETECTION}" = "true" ] + then + echo "Setting dockerd Azure DNS." + CUSTOMDNS="--dns 168.63.129.16" + else + echo "Not setting dockerd DNS manually." + CUSTOMDNS="" + fi + set -e + + if [ -z "$DOCKER_DEFAULT_ADDRESS_POOL" ] then - echo "Setting dockerd Azure DNS." - CUSTOMDNS="--dns 168.63.129.16" + DEFAULT_ADDRESS_POOL="" else - echo "Not setting dockerd DNS manually." - CUSTOMDNS="" + DEFAULT_ADDRESS_POOL="--default-address-pool $DOCKER_DEFAULT_ADDRESS_POOL" fi - set -e # Start docker/moby engine - ( dockerd $CUSTOMDNS > /tmp/dockerd.log 2>&1 ) & + ( dockerd $CUSTOMDNS $DEFAULT_ADDRESS_POOL $DOCKER_DEFAULT_IP6_TABLES > /tmp/dockerd.log 2>&1 ) & INNEREOF )" -# Start using sudo if not invoked as root -if [ "$(id -u)" -ne 0 ]; then - sudo /bin/sh -c "${dockerd_start}" -else - eval "${dockerd_start}" -fi +sudo_if() { + COMMAND="$*" + + if [ "$(id -u)" -ne 0 ]; then + sudo $COMMAND + else + $COMMAND + fi +} -set +e +retry_docker_start_count=0 +docker_ok="false" + +until [ "${docker_ok}" = "true" ] || [ "${retry_docker_start_count}" -eq "5" ]; +do + # Start using sudo if not invoked as root + if [ "$(id -u)" -ne 0 ]; then + sudo /bin/sh -c "${dockerd_start}" + else + eval "${dockerd_start}" + fi + + retry_count=0 + until [ "${docker_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + sleep 1s + set +e + docker info > /dev/null 2>&1 && docker_ok="true" + set -e + + retry_count=`expr $retry_count + 1` + done + + if [ "${docker_ok}" != "true" ] && [ "${retry_docker_start_count}" != "4" ]; then + echo "(*) Failed to start docker, retrying..." + set +e + sudo_if pkill dockerd + sudo_if pkill containerd + set -e + fi + + retry_docker_start_count=`expr $retry_docker_start_count + 1` +done # Execute whatever commands were passed in (if any). This allows us # to set this script to ENTRYPOINT while still executing the default CMD. @@ -394,4 +637,7 @@ EOF chmod +x /usr/local/share/docker-init.sh chown ${USERNAME}:root /usr/local/share/docker-init.sh -echo 'docker-in-docker-debian script has completed!' +# Clean up +rm -rf /var/lib/apt/lists/* + +echo 'docker-in-docker-debian script has completed!' \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 467d7374fadfa..db2db10db7f41 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,6 +17,7 @@ !.kearc !.storybook !tsconfig.json +!products !frontend/@posthog !frontend/src !frontend/types @@ -32,10 +33,10 @@ !plugin-server/.eslintrc.js !plugin-server/.prettierrc !share/GeoLite2-City.mmdb -!hogvm/python +!common/hogvm/python !unit.json.tpl -!plugin-transpiler/src -!plugin-transpiler/*.* +!common/plugin_transpiler/src +!common/plugin_transpiler/*.* !test-runner-jest.config.js !test-runner-jest-environment.js !patches diff --git a/.eslintignore b/.eslintignore index 29d959598349e..0f47ab7fcf055 100644 --- a/.eslintignore +++ b/.eslintignore @@ -3,3 +3,8 @@ jest.config.ts node_modules rust livestream +common/hogvm/__tests__/__snapshots__ +common/hogvm/typescript/dist +common/hogvm/typescript/jest.config.js +common/plugin_transpiler/dist +common/plugin_transpiler/build.mjs diff --git a/.flox/env/manifest.lock b/.flox/env/manifest.lock index 2fa6bf0f4d140..cc61f70b0cf27 100644 --- a/.flox/env/manifest.lock +++ b/.flox/env/manifest.lock @@ -71,8 +71,7 @@ }, "xmlsec": { "pkg-path": "xmlsec", - "pkg-group": "python", - "version": "1.2.34" + "version": "1.3.6" } }, "vars": { @@ -593,130 +592,6 @@ "group": "python", "priority": 5 }, - { - "attr_path": "xmlsec", - "broken": false, - "derivation": "/nix/store/p85izfrfdj8n22pm9iv1b8a511pm2i5n-xmlsec-1.2.34.drv", - "description": "XML Security Library in C based on libxml2", - "install_id": "xmlsec", - "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", - "name": "xmlsec-1.2.34", - "pname": "xmlsec", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "1.2.34", - "outputs_to_install": [ - "out" - ], - "outputs": { - "dev": "/nix/store/mnrmn6vndrfgxyxmbqvmrwi7qjwr3d0m-xmlsec-1.2.34-dev", - "out": "/nix/store/ff6sr1v2q8gqmnyvpl6xfxw3r2lb56i3-xmlsec-1.2.34" - }, - "system": "aarch64-darwin", - "group": "python", - "priority": 5 - }, - { - "attr_path": "xmlsec", - "broken": false, - "derivation": "/nix/store/7zscs792mb7v53i43jf4qq0gypgrql8g-xmlsec-1.2.34.drv", - "description": "XML Security Library in C based on libxml2", - "install_id": "xmlsec", - "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", - "name": "xmlsec-1.2.34", - "pname": "xmlsec", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "1.2.34", - "outputs_to_install": [ - "out" - ], - "outputs": { - "dev": "/nix/store/k8n1j04qrbhlzxklrxhk32aphh58jqcf-xmlsec-1.2.34-dev", - "out": "/nix/store/qdscjbwglk2953azhhhsibf0xlsjig1h-xmlsec-1.2.34" - }, - "system": "aarch64-linux", - "group": "python", - "priority": 5 - }, - { - "attr_path": "xmlsec", - "broken": false, - "derivation": "/nix/store/cbl3q86h6vzzzi50ph82syrkha9g9mah-xmlsec-1.2.34.drv", - "description": "XML Security Library in C based on libxml2", - "install_id": "xmlsec", - "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", - "name": "xmlsec-1.2.34", - "pname": "xmlsec", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "1.2.34", - "outputs_to_install": [ - "out" - ], - "outputs": { - "dev": "/nix/store/jc3nxrj9kv447gb5ywxqal45b7ick5c7-xmlsec-1.2.34-dev", - "out": "/nix/store/fzhimicsixlzc8k6l0f5vh5lzbxsl39c-xmlsec-1.2.34" - }, - "system": "x86_64-darwin", - "group": "python", - "priority": 5 - }, - { - "attr_path": "xmlsec", - "broken": false, - "derivation": "/nix/store/5z6p8lb90900s6k8lml62mlwjjx6whm2-xmlsec-1.2.34.drv", - "description": "XML Security Library in C based on libxml2", - "install_id": "xmlsec", - "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", - "name": "xmlsec-1.2.34", - "pname": "xmlsec", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", - "stabilities": [ - "staging", - "unstable" - ], - "unfree": false, - "version": "1.2.34", - "outputs_to_install": [ - "out" - ], - "outputs": { - "dev": "/nix/store/z0k51x5cg15f2r058ypjzhf9bakzq16g-xmlsec-1.2.34-dev", - "out": "/nix/store/rrnig5ybklsrf8kiwn15lvci0rfq6379-xmlsec-1.2.34" - }, - "system": "x86_64-linux", - "group": "python", - "priority": 5 - }, { "attr_path": "cargo", "broken": false, @@ -1357,17 +1232,17 @@ { "attr_path": "mprocs", "broken": false, - "derivation": "/nix/store/5hckam5lgqqmlkjd1r6a0bzkkvhfm349-mprocs-0.7.1.drv", + "derivation": "/nix/store/26d7rl3vql93s040slf69jw5a4wxflhj-mprocs-0.7.1.drv", "description": "TUI tool to run multiple commands in parallel and show the output of each command separately", "install_id": "mprocs", "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", "name": "mprocs-0.7.1", "pname": "mprocs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", "stabilities": [ "staging", "unstable" @@ -1378,7 +1253,7 @@ "out" ], "outputs": { - "out": "/nix/store/pk1mb41ryrw2izq15pymxjwqvw8mr29p-mprocs-0.7.1" + "out": "/nix/store/6qd6isn3scvlp7vk34xgb1mn9hh93p8v-mprocs-0.7.1" }, "system": "aarch64-darwin", "group": "toplevel", @@ -1387,17 +1262,17 @@ { "attr_path": "mprocs", "broken": false, - "derivation": "/nix/store/lg4yvv4az18g59l7rgfvn2pqxidx5qgb-mprocs-0.7.1.drv", + "derivation": "/nix/store/bi3z98bxszfz04qdpyxkmrcjkq75iasa-mprocs-0.7.1.drv", "description": "TUI tool to run multiple commands in parallel and show the output of each command separately", "install_id": "mprocs", "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", "name": "mprocs-0.7.1", "pname": "mprocs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", "stabilities": [ "staging", "unstable" @@ -1408,7 +1283,7 @@ "out" ], "outputs": { - "out": "/nix/store/dynm867fg992nb5wlysvrg1p14cwgsca-mprocs-0.7.1" + "out": "/nix/store/4sdqi7adqhxqvyii5n8a1gpg740hbshm-mprocs-0.7.1" }, "system": "aarch64-linux", "group": "toplevel", @@ -1417,17 +1292,17 @@ { "attr_path": "mprocs", "broken": false, - "derivation": "/nix/store/xbicacfnn0gnz08rri7jwc40byxrm404-mprocs-0.7.1.drv", + "derivation": "/nix/store/6qmqgqkjbdsn5z308hp3w84wav6p1phb-mprocs-0.7.1.drv", "description": "TUI tool to run multiple commands in parallel and show the output of each command separately", "install_id": "mprocs", "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", "name": "mprocs-0.7.1", "pname": "mprocs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", "stabilities": [ "staging", "unstable" @@ -1438,7 +1313,7 @@ "out" ], "outputs": { - "out": "/nix/store/kx138kp8rxjwph6wqp3lk8v6fybij2bz-mprocs-0.7.1" + "out": "/nix/store/cfsrk2za8x6d6n5wdwjwpm9hf6ws28qi-mprocs-0.7.1" }, "system": "x86_64-darwin", "group": "toplevel", @@ -1447,17 +1322,17 @@ { "attr_path": "mprocs", "broken": false, - "derivation": "/nix/store/ydrq0m8lp0m3pjpd9ndg1x30z3bg81qx-mprocs-0.7.1.drv", + "derivation": "/nix/store/hlmqv197aqi8imigjj061mdmhbij7v94-mprocs-0.7.1.drv", "description": "TUI tool to run multiple commands in parallel and show the output of each command separately", "install_id": "mprocs", "license": "MIT", - "locked_url": "https://github.com/flox/nixpkgs?rev=9f4128e00b0ae8ec65918efeba59db998750ead6", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", "name": "mprocs-0.7.1", "pname": "mprocs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", - "rev_count": 647193, - "rev_date": "2024-07-03T18:27:49Z", - "scrape_date": "2024-07-05T00:14:29Z", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", "stabilities": [ "staging", "unstable" @@ -1468,7 +1343,131 @@ "out" ], "outputs": { - "out": "/nix/store/ky3hbhjvsr4l5nh1xjipkbswll37j0dq-mprocs-0.7.1" + "out": "/nix/store/bhc6rvwb1n4bh8rsxwnw7s9krlphqkh4-mprocs-0.7.1" + }, + "system": "x86_64-linux", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "xmlsec", + "broken": false, + "derivation": "/nix/store/npf1l5rkpmp5swm4jrfypx975wrmjzc8-xmlsec-1.3.6.drv", + "description": "XML Security Library in C based on libxml2", + "install_id": "xmlsec", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", + "name": "xmlsec-1.3.6", + "pname": "xmlsec", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "1.3.6", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/vj56i5ldpvxfhh7fzh30fsj925rv08a6-xmlsec-1.3.6-dev", + "out": "/nix/store/66can923s5d00xgzzmlbxz2ydgmgglcb-xmlsec-1.3.6" + }, + "system": "aarch64-darwin", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "xmlsec", + "broken": false, + "derivation": "/nix/store/l54yryhpx0wfm8sv88ys92z9i0hsip3k-xmlsec-1.3.6.drv", + "description": "XML Security Library in C based on libxml2", + "install_id": "xmlsec", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", + "name": "xmlsec-1.3.6", + "pname": "xmlsec", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "1.3.6", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/fscw35frjnm34ra3l3jyycfivn20r9va-xmlsec-1.3.6-dev", + "out": "/nix/store/18l3z48p7w2h3anq8zmgrhqp5j25s5g9-xmlsec-1.3.6" + }, + "system": "aarch64-linux", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "xmlsec", + "broken": false, + "derivation": "/nix/store/jcjvbiyzq7648zs65v318qzb6hcvp038-xmlsec-1.3.6.drv", + "description": "XML Security Library in C based on libxml2", + "install_id": "xmlsec", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", + "name": "xmlsec-1.3.6", + "pname": "xmlsec", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "1.3.6", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/brmp8jjpi2x6nzxfwq7czr46fjhif8c2-xmlsec-1.3.6-dev", + "out": "/nix/store/6q35gr9jggmql201i2iz9gjkbfj30lf7-xmlsec-1.3.6" + }, + "system": "x86_64-darwin", + "group": "toplevel", + "priority": 5 + }, + { + "attr_path": "xmlsec", + "broken": false, + "derivation": "/nix/store/7c8zh84rs39q6p5bm10dvwmbzcf96i4c-xmlsec-1.3.6.drv", + "description": "XML Security Library in C based on libxml2", + "install_id": "xmlsec", + "license": "MIT", + "locked_url": "https://github.com/flox/nixpkgs?rev=8f3e1f807051e32d8c95cd12b9b421623850a34d", + "name": "xmlsec-1.3.6", + "pname": "xmlsec", + "rev": "8f3e1f807051e32d8c95cd12b9b421623850a34d", + "rev_count": 733374, + "rev_date": "2025-01-04T17:41:09Z", + "scrape_date": "2025-01-06T21:17:53Z", + "stabilities": [ + "staging", + "unstable" + ], + "unfree": false, + "version": "1.3.6", + "outputs_to_install": [ + "out" + ], + "outputs": { + "dev": "/nix/store/xixr2nq7pjbbl8mn43c12m2m8gvl4fhq-xmlsec-1.3.6-dev", + "out": "/nix/store/3308q19l44wkil15x2zy821dlz4ig44f-xmlsec-1.3.6" }, "system": "x86_64-linux", "group": "toplevel", diff --git a/.flox/env/manifest.toml b/.flox/env/manifest.toml index 6a3996a00c79e..6af0be27dd18e 100644 --- a/.flox/env/manifest.toml +++ b/.flox/env/manifest.toml @@ -12,7 +12,6 @@ version = 1 # Python python3 = { pkg-path = "python3", version = "3.11", pkg-group = "python" } uv = { pkg-path = "uv", pkg-group = "python" } -xmlsec = { pkg-path = "xmlsec", pkg-group = "python", version = "1.2.34" } libtool = { pkg-path = "libtool", pkg-group = "python" } # Node nodejs = { pkg-path = "nodejs_18", pkg-group = "nodejs" } @@ -38,6 +37,8 @@ libiconv.pkg-group = "rust-toolchain" go = { pkg-path = "go", version = "1.22", pkg-group = "go" } # General CLI tools mprocs.pkg-path = "mprocs" +xmlsec.pkg-path = "xmlsec" +xmlsec.version = "1.3.6" # Set environment variables in the `[vars]` section. These variables may not # reference one another, and are added to the environment without first diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 5bcacc4be27d6..f584005d8d5f6 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -19,4 +19,4 @@ body: - type: markdown attributes: - value: '#### *Thank you* for your bug report – we love squashing them!' + value: '#### *Thank you* for your feature request – we love each and every one!' diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 7cfa27aacccf8..c5cf1237bd03b 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -63,7 +63,7 @@ runs: id: hogql-parser-diff run: | git fetch --no-tags --prune --depth=1 origin master - changed=$(git diff --quiet HEAD origin/master -- hogql_parser/ && echo "false" || echo "true") + changed=$(git diff --quiet HEAD origin/master -- common/hogql_parser/ && echo "false" || echo "true") echo "changed=$changed" >> $GITHUB_OUTPUT - name: Install SAML (python3-saml) dependencies @@ -86,10 +86,10 @@ runs: # absolute wild tbh https://stackoverflow.com/a/75503402 - uses: tlambert03/setup-qt-libs@v1 - - name: Install plugin-transpiler + - name: Install plugin_transpiler shell: bash run: | - cd plugin-transpiler + cd common/plugin_transpiler pnpm install pnpm run build @@ -122,7 +122,7 @@ runs: sudo cp out/usr/local/lib/libantlr4-runtime.so* /usr/lib/ sudo ldconfig cd .. - uv pip install --system ./hogql_parser + uv pip install --system ./common/hogql_parser - name: Set up needed files shell: bash @@ -164,7 +164,7 @@ runs: || 'posthog' }} ${{ inputs.person-on-events == 'true' && 'ee/clickhouse/' || 'ee/' }} -m "not async_migrations" \ --ignore=posthog/temporal \ - --ignore=hogvm/python/test \ + --ignore=common/hogvm/python/test \ --splits ${{ inputs.concurrency }} --group ${{ inputs.group }} \ --durations=100 --durations-min=1.0 --store-durations \ $PYTEST_ARGS diff --git a/.github/pr-deploy/hobby.yaml.tmpl b/.github/pr-deploy/hobby.yaml.tmpl index f284937c6a2a5..487287f579603 100644 --- a/.github/pr-deploy/hobby.yaml.tmpl +++ b/.github/pr-deploy/hobby.yaml.tmpl @@ -20,8 +20,8 @@ spec: privileged: true resources: requests: - cpu: 2 - memory: 500M + cpu: 4 + memory: 4G ports: - containerPort: 2375 - containerPort: 80 diff --git a/.github/workflows/build-hogql-parser.yml b/.github/workflows/build-hogql-parser.yml index 8feffe960e335..d302c4b3ebd11 100644 --- a/.github/workflows/build-hogql-parser.yml +++ b/.github/workflows/build-hogql-parser.yml @@ -3,7 +3,7 @@ name: Release hogql-parser on: pull_request: paths: - - hogql_parser/** + - common/hogql_parser/** - .github/workflows/build-hogql-parser.yml concurrency: @@ -21,14 +21,14 @@ jobs: with: fetch-depth: 0 # Fetching all for comparison since last push (not just last commit) - - name: Check if hogql_parser/ has changed + - name: Check if common/hogql_parser/ has changed id: changed-files uses: tj-actions/changed-files@v43 with: since_last_remote_commit: true files_yaml: | parser: - - hogql_parser/** + - common/hogql_parser/** - name: Check if version was bumped shell: bash @@ -37,7 +37,7 @@ jobs: parser_release_needed='false' if [[ ${{ steps.changed-files.outputs.parser_any_changed }} == 'true' ]]; then published=$(curl -fSsl https://pypi.org/pypi/hogql-parser/json | jq -r '.info.version') - local=$(python hogql_parser/setup.py --version) + local=$(python common/hogql_parser/setup.py --version) if [[ "$published" != "$local" ]]; then parser_release_needed='true' else @@ -59,48 +59,34 @@ jobs: # As of October 2023, GitHub doesn't have ARM Actions runners… and ARM emulation is insanely slow # (20x longer) on the Linux runners (while being reasonable on the macOS runners). Hence, we use # BuildJet as a provider of ARM runners - this solution saves a lot of time and consequently some money. - os: [ubuntu-22.04, buildjet-2vcpu-ubuntu-2204-arm, macos-12] + os: [ubuntu-22.04, buildjet-2vcpu-ubuntu-2204-arm, macos-13] steps: - uses: actions/checkout@v4 - - if: ${{ !endsWith(matrix.os, '-arm') }} - uses: actions/setup-python@v4 + - uses: actions/setup-python@v4 with: python-version: '3.11' - # Compiling Python 3.11 from source on ARM. We tried using the "deadsnakes" ARM repo, but it was flakey. - - if: ${{ endsWith(matrix.os, '-arm') }} - name: Install Python 3.11 on ARM (compile from source) - run: | - sudo apt-get update - sudo apt-get install -y build-essential libssl-dev zlib1g-dev \ - libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev \ - libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev - wget https://www.python.org/ftp/python/3.11.0/Python-3.11.0.tar.xz - tar -xf Python-3.11.0.tar.xz - cd Python-3.11.0 - ./configure --enable-optimizations - make -j 2 - sudo make altinstall - - name: Build sdist if: matrix.os == 'ubuntu-22.04' # Only build the sdist once - run: cd hogql_parser && python setup.py sdist + run: cd common/hogql_parser && python setup.py sdist - name: Install cibuildwheel run: pip install cibuildwheel==2.16.* - name: Build wheels - run: cd hogql_parser && python -m cibuildwheel --output-dir dist + run: cd common/hogql_parser && python -m cibuildwheel --output-dir dist env: - MACOSX_DEPLOYMENT_TARGET: '12' # A modern target allows us to use C++20 + MACOSX_DEPLOYMENT_TARGET: '13' # A modern target allows us to use C++20 - - uses: actions/upload-artifact@v4 + - name: Upload wheels artifact + uses: actions/upload-artifact@v4 with: + name: wheels-${{ matrix.os }} path: | - hogql_parser/dist/*.whl - hogql_parser/dist/*.tar.gz + common/hogql_parser/dist/*.whl + common/hogql_parser/dist/*.tar.gz if-no-files-found: error publish: @@ -111,11 +97,23 @@ jobs: id-token: write runs-on: ubuntu-22.04 steps: - - name: Fetch wheels + - name: Download wheels from ubuntu-22.04 + uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-22.04 + path: dist + + - name: Download wheels from buildjet-2vcpu-ubuntu-2204-arm + uses: actions/download-artifact@v4 + with: + name: wheels-buildjet-2vcpu-ubuntu-2204-arm + path: dist + + - name: Download wheels from macos-13 uses: actions/download-artifact@v4 with: - name: artifact - path: dist/ + name: wheels-macos-13 + path: dist - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -128,7 +126,7 @@ jobs: - name: Update hogql-parser in requirements shell: bash run: | - local=$(python hogql_parser/setup.py --version) + local=$(python common/hogql_parser/setup.py --version) sed -i "s/hogql-parser==.*/hogql-parser==${local}/g" requirements.in sed -i "s/hogql-parser==.*/hogql-parser==${local}/g" requirements.txt diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 22882cafb7d09..bd2fff04f1268 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -68,7 +68,7 @@ jobs: # including the negated rule appears to work # but makes it always match because the checked file always isn't `ee/frontend/**` 🙈 - 'ee/**/*' - - 'hogvm/**/*' + - 'common/hogvm/**/*' - 'posthog/**/*' - 'bin/*.py' - requirements.txt @@ -76,7 +76,7 @@ jobs: - mypy.ini - pytest.ini - frontend/src/queries/schema.json # Used for generating schema.py - - plugin-transpiler/src # Used for transpiling plugins + - common/plugin_transpiler/src # Used for transpiling plugins # Make sure we run if someone is explicitly change the workflow - .github/workflows/ci-backend.yml - .github/actions/run-backend-tests/action.yml @@ -316,7 +316,7 @@ jobs: uses: actions/upload-artifact@v4 if: needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' && matrix.person-on-events == false with: - name: email_renders + name: email_renders-${{ matrix.segment }}-${{ matrix.person-on-events }} path: posthog/tasks/test/__emails__ retention-days: 5 diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 8717352036230..443251d305d3e 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -242,21 +242,21 @@ jobs: - name: Archive test screenshots uses: actions/upload-artifact@v4 with: - name: screenshots + name: screenshots-${{ strategy.job-index }} path: cypress/screenshots if: ${{ failure() }} - name: Archive test downloads uses: actions/upload-artifact@v4 with: - name: downloads + name: downloads-${{ strategy.job-index }} path: cypress/downloads if: ${{ failure() }} - name: Archive test videos uses: actions/upload-artifact@v4 with: - name: videos + name: videos-${{ strategy.job-index }} path: cypress/videos if: ${{ failure() }} @@ -264,7 +264,7 @@ jobs: if: needs.changes.outputs.shouldTriggerCypress == 'true' uses: actions/upload-artifact@v4 with: - name: accessibility-violations + name: accessibility-violations-${{ strategy.job-index }} path: '**/a11y/' if-no-files-found: 'ignore' diff --git a/.github/workflows/ci-hog.yml b/.github/workflows/ci-hog.yml index ea51f70721f5c..89bed9a7295a0 100644 --- a/.github/workflows/ci-hog.yml +++ b/.github/workflows/ci-hog.yml @@ -37,7 +37,7 @@ jobs: filters: | hog: # Avoid running tests for irrelevant changes - - 'hogvm/**/*' + - 'common/hogvm/**/*' - 'posthog/hogql/**/*' - 'bin/hog' - 'bin/hoge' @@ -111,27 +111,27 @@ jobs: npm run grammar:build && git diff --exit-code env: # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2), - # as apt-get is quite out of date. The same version must be set in hogql_parser/pyproject.toml + # as apt-get is quite out of date. The same version must be set in common/hogql_parser/pyproject.toml ANTLR_VERSION: '4.13.2' - name: Check if STL bytecode is up to date run: | - python -m hogvm.stl.compile + python -m common.hogvm.stl.compile git diff --exit-code - name: Run HogVM Python tests run: | - pytest hogvm + pytest common/hogvm - name: Run HogVM TypeScript tests run: | - cd hogvm/typescript + cd common/hogvm/typescript pnpm install --frozen-lockfile pnpm run test - name: Run Hog tests run: | - cd hogvm/typescript + cd common/hogvm/typescript pnpm run build cd ../ ./test.sh && git diff --exit-code @@ -152,7 +152,7 @@ jobs: id: check-package-version uses: PostHog/check-package-version@v2 with: - path: hogvm/typescript + path: common/hogvm/typescript release-hogvm: name: Release new HogVM TypeScript version @@ -191,9 +191,9 @@ jobs: node-version: 18 registry-url: https://registry.npmjs.org - name: Install package.json dependencies - run: cd hogvm/typescript && pnpm install + run: cd common/hogvm/typescript && pnpm install - name: Publish the package in the npm registry - run: cd hogvm/typescript && npm publish --access public + run: cd common/hogvm/typescript && npm publish --access public env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - name: Sleep 60 seconds to allow npm to update the package @@ -225,7 +225,7 @@ jobs: id: check-mismatch run: | # Extract committed version - HOGVM_VERSION=$(jq -r '.version' hogvm/typescript/package.json) + HOGVM_VERSION=$(jq -r '.version' common/hogvm/typescript/package.json) # Compare dependencies in package.json MAIN_VERSION=$(jq -r '.dependencies."@posthog/hogvm"' package.json | tr -d '^') @@ -245,7 +245,7 @@ jobs: - name: Update package.json versions if: env.mismatch == 'true' run: | - VERSION=$(jq ".version" hogvm/typescript/package.json -r) + VERSION=$(jq ".version" common/hogvm/typescript/package.json -r) retry_pnpm_install() { local retries=0 @@ -283,7 +283,7 @@ jobs: if: env.mismatch == 'true' uses: EndBug/add-and-commit@v9 with: - add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "hogvm/typescript/package.json"]' + add: '["package.json", "pnpm-lock.yaml", "plugin-server/package.json", "plugin-server/pnpm-lock.yaml", "common/hogvm/typescript/package.json"]' message: 'Update @posthog/hogvm version in package.json' default_author: github_actions github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/ci-plugin-server.yml b/.github/workflows/ci-plugin-server.yml index 1c8ba97095f63..79890fa66cff8 100644 --- a/.github/workflows/ci-plugin-server.yml +++ b/.github/workflows/ci-plugin-server.yml @@ -188,7 +188,7 @@ jobs: DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/test_posthog' REDIS_URL: 'redis://localhost' NODE_OPTIONS: '--max_old_space_size=4096' - run: cd plugin-server && pnpm test -- --runInBand --forceExit tests/ --shard=${{matrix.shard}} + run: cd plugin-server && pnpm test -- --runInBand --forceExit --shard=${{matrix.shard}} functional-tests: name: Functional tests diff --git a/.github/workflows/rust-docker-build.yml b/.github/workflows/rust-docker-build.yml index 4237599d9b311..02622c99a6207 100644 --- a/.github/workflows/rust-docker-build.yml +++ b/.github/workflows/rust-docker-build.yml @@ -33,6 +33,8 @@ jobs: dockerfile: ./rust/Dockerfile - image: cymbal dockerfile: ./rust/Dockerfile + - image: batch-import-worker + dockerfile: ./rust/Dockerfile runs-on: depot-ubuntu-22.04-4 permissions: id-token: write # allow issuing OIDC tokens for this workflow run @@ -44,6 +46,7 @@ jobs: cyclotron-fetch_digest: ${{ steps.digest.outputs.cyclotron-fetch_digest }} cyclotron-janitor_digest: ${{ steps.digest.outputs.cyclotron-janitor_digest }} property-defs-rs_digest: ${{ steps.digest.outputs.property-defs-rs_digest }} + batch-import-worker_digest: ${{ steps.digest.outputs.batch-import-worker_digest }} hook-api_digest: ${{ steps.digest.outputs.hook-api_digest }} hook-janitor_digest: ${{ steps.digest.outputs.hook-janitor_digest }} hook-worker_digest: ${{ steps.digest.outputs.hook-worker_digest }} @@ -142,6 +145,10 @@ jobs: values: image: sha: '${{ needs.build.outputs.property-defs-rs_digest }}' + - release: batch-import-worker + values: + image: + sha: '${{ needs.build.outputs.batch-import-worker_digest }}' - release: cymbal values: image: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 29268601c4db4..ea90be36f43b2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -83,7 +83,7 @@ jobs: - feature-flags - others needs: changes - runs-on: depot-ubuntu-22.04-4 + runs-on: depot-ubuntu-24.04-4 timeout-minutes: 10 defaults: diff --git a/.gitignore b/.gitignore index cf3a2812844f3..cf20b572e03d7 100644 --- a/.gitignore +++ b/.gitignore @@ -56,10 +56,10 @@ gen/ # Local files generated for the ANTLR VS Code extension (https://github.com/mike-lischke/vscode-antlr4) .antlr upgrade/ -hogvm/typescript/dist -hogvm/typescript/.parcel-cache +common/hogvm/typescript/dist +common/hogvm/typescript/.parcel-cache .wokeignore -plugin-transpiler/dist +common/plugin_transpiler/dist *-esbuild-meta.json *-esbuild-bundle-visualization.html .dlt diff --git a/.prettierignore b/.prettierignore index 121a272bb0b16..890b3b1c5adb4 100644 --- a/.prettierignore +++ b/.prettierignore @@ -17,6 +17,6 @@ dist/ node_modules/ pnpm-lock.yaml posthog/templates/email/* -hogvm/typescript/src/stl/bytecode.ts +common/hogvm/typescript/src/stl/bytecode.ts rust/ livestream/ diff --git a/.run/PostHog.run.xml b/.run/PostHog.run.xml index d006dc469d4a0..45e841bf25319 100644 --- a/.run/PostHog.run.xml +++ b/.run/PostHog.run.xml @@ -1,7 +1,7 @@ -