diff --git a/.cirrus.yml b/.cirrus.yml index 1767c8b..e9c5f8a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -20,6 +20,10 @@ env: # GCP_REGION # GCP_REPO + # It is easier to manage VCKPG_PACKAGES in a single place + VCPKG_PACKAGES: | + gettext icu libxml2 libxslt lz4 openssl pkgconf python3 tcl zlib zstd + # central definition for the tasks interacting with gcp @@ -101,27 +105,36 @@ task: name: 'Building VM Image: ${IMAGE_NAME}' alias: vmbuild-${TASK_NAME} matrix: - - env: - TASK_NAME: freebsd-13 - PACKERFILE: packer/freebsd.pkr.hcl - - - env: - PACKERFILE: packer/linux_debian.pkr.hcl - SCRIPTS: scripts/linux_debian_* - + - depends_on: cleanup-leftover-running matrix: - env: - TASK_NAME: bookworm - - env: - TASK_NAME: bullseye - - env: - TASK_NAME: sid + TASK_NAME: freebsd-13 + PACKERFILE: packer/freebsd.pkr.hcl + - env: - TASK_NAME: sid-newkernel + PACKERFILE: packer/linux_debian.pkr.hcl + SCRIPTS: scripts/linux_debian_* + + matrix: + - env: + TASK_NAME: bookworm + - env: + TASK_NAME: bullseye + - env: + TASK_NAME: sid + - env: + TASK_NAME: sid-newkernel + - env: + TASK_NAME: sid-newkernel-uring + - env: - TASK_NAME: sid-newkernel-uring + PACKERFILE: packer/windows_base.pkr.hcl + TASK_NAME: windows-base + SCRIPTS: scripts/windows_install_vs_2019.ps1 - - env: + # build-vcpkg-cache task already depends on cleanup-leftover-running task + - depends_on: build-vcpkg-cache + env: PACKERFILE: packer/windows.pkr.hcl SCRIPTS: scripts/windows* TASK_NAME: windows-ci @@ -129,8 +142,6 @@ task: env: IMAGE_NAME: ${PREFIX}-${TASK_NAME} - depends_on: cleanup-leftover-running - container: dockerfile: docker/linux_debian_packer cpu: 0.5 @@ -172,6 +183,67 @@ task: ${IMAGE_NAME}-${DATE} +task: + name: 'Build VCPKG Cache' + alias: build-vcpkg-cache + depends_on: vmbuild-windows-base + + env: + VCPKG_PATH: ${CIRRUS_WORKING_DIR}\vcpkg + VCPKG_CACHE: ${VCPKG_PATH}\binary_cache + CIRRUS_SHELL: powershell + ErrorActionPreference: Stop + + compute_engine_instance: + image_project: $GCP_PROJECT + image: family/pg-ci-windows-base + platform: windows + cpu: 4 + disk: 50 + + sysinfo_script: | + chcp + systeminfo + get-psdrive -psprovider filesystem + Get-ChildItem -Path Env: + + clone_vcpkg_script: + git clone --depth 1 https://github.com/Microsoft/vcpkg.git ${Env:VCPKG_PATH} + + vcpkg_cache: + folder: ${VCPKG_CACHE} + + install_vcpkg_packages_script: | + cd ${Env:VCPKG_PATH} + .\bootstrap-vcpkg.bat -disableMetrics + .\vcpkg.exe install --debug --binarysource=files,${Env:VCPKG_CACHE},readwrite ${Env:VCPKG_PACKAGES} --triplet=x64-windows + if (!$?) { throw 'cmdfail' } + + upload_caches: vcpkg + + zip_vcpkg_cache_script: | + 7z.exe a -r vcpkg_cache.zip ${Env:VCPKG_CACHE}\* + + vcpkg_cache_zip_artifacts: + path: vcpkg_cache.zip + + +task: + name: Clean up Windows base images + depends_on: vmbuild-windows-ci + + container: + dockerfile: docker/linux_debian_packer + cpu: 0.5 + memory: 256Mi + + <<: *gcp_auth_unix + + # There could be leftover windows-base images, so delete all of them + cleanup_windows_base_images_script: | + export IMAGES=$(gcloud compute images list --project $GCP_PROJECT --format='object value(name)' --no-standard-images | grep "$PREFIX-windows-base") + gcloud compute images delete --quiet --project $GCP_PROJECT $IMAGES + task: name: 'Building VM Image: ${IMAGE_NAME}' alias: vmbuild-${TASK_NAME} diff --git a/Makefile b/Makefile index a045f92..7ca3767 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,12 @@ pre-commit: -var "image_date=$(IMAGE_DATE)" \ -var "image_name=bookworm" \ packer/linux_debian.pkr.hcl +# Windows Base VM + packer validate \ + -var gcp_project=pg-ci-images-dev \ + -var "image_date=$(IMAGE_DATE)" \ + -var "image_name=windows-ci-base" \ + packer/windows_base.pkr.hcl # Windows VM packer validate \ -var gcp_project=pg-ci-images-dev \ diff --git a/packer/windows.pkr.hcl b/packer/windows.pkr.hcl index 8f8f223..e9611d5 100644 --- a/packer/windows.pkr.hcl +++ b/packer/windows.pkr.hcl @@ -1,9 +1,12 @@ variable "image_name" { type = string } variable "image_date" { type = string } -variable "build_type" { - type = string - default = "googlecompute" +variable "cirrus_build_id" { + default = env("CIRRUS_BUILD_ID") +} + +variable "VCPKG_PACKAGES" { + default = env("VCPKG_PACKAGES") } # Packer doesn't capture errors correctly when default execute command is used. @@ -23,7 +26,6 @@ locals { image_identity = "${var.image_name}-${var.image_date}" perl_version = "5.26.3.1" - python_version = "3.10.6" windows_gcp_images = [ { @@ -36,7 +38,7 @@ source "googlecompute" "windows" { disk_size = "50" disk_type = "pd-ssd" project_id = var.gcp_project - source_image_family = "windows-2022" + source_image_family = "pg-ci-windows-base" image_name = local.image_identity zone = "us-west1-a" machine_type = "t2d-standard-4" @@ -59,7 +61,7 @@ build { # so we can easily combine the packer invocations later dynamic "source" { for_each = local.windows_gcp_images - labels = ["source.${var.build_type}.windows"] + labels = ["source.googlecompute.windows"] iterator = tag content { @@ -67,30 +69,9 @@ build { } } - ### base installations - # googlecompute only provisioner "powershell" { execute_command = var.execute_command inline = [ - "$ErrorActionPreference = 'Stop'", - # disable antivirus - "Set-MpPreference -DisableRealtimeMonitoring $true -SubmitSamplesConsent NeverSend -MAPSReporting Disable", - - # install choco - "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))", - - "choco install -y --no-progress 7zip", - "choco install -y --no-progress git --parameters=\"/GitAndUnixToolsOnPath\"", - ] - } - - provisioner "powershell" { - execute_command = var.execute_command - inline = [ - "$ErrorActionPreference = 'Stop'", - # contains useful utilities, including a diff we can use - "[Environment]::SetEnvironmentVariable('PATH', 'C:\\Program Files\\Git\\usr\\bin;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine')", - # set IMAGE_IDENTITY to distinguish images on CI runs "[Environment]::SetEnvironmentVariable('IMAGE_IDENTITY', '${local.image_identity}', 'Machine')", ] @@ -102,19 +83,25 @@ build { script = "scripts/windows_install_dbg.ps1" } - # install python + ### vs-2019 installations + provisioner "powershell" { + execute_command = var.execute_command + script = "scripts/windows_install_winflexbison.ps1" + } + provisioner "powershell" { execute_command = var.execute_command - environment_vars = ["TEMP_PYTHON_VERSION=${local.python_version}"] - script = "scripts/windows_install_python.ps1" + environment_vars = ["CIRRUS_BUILD_ID=${var.cirrus_build_id}", + "VCPKG_PACKAGES=${var.VCPKG_PACKAGES}"] + script = "scripts/windows_install_pg_deps.ps1" } - # install meson and ninja provisioner "powershell" { execute_command = var.execute_command inline = [ "$ErrorActionPreference = 'Stop'", - "py -m pip install meson ninja" + "python3 -m ensurepip", + "python3 -m pip install meson ninja" ] } @@ -134,13 +121,7 @@ build { "[Environment]::SetEnvironmentVariable('PATH', \"C:\\strawberry\\${local.perl_version}\\perl\\bin;\" + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine')", ] } - - # install openssl - provisioner "powershell" { - execute_command = var.execute_command - script = "scripts/windows_install_openssl.ps1" - } - ### end of base installations + ### end of vs-2019 installations ### mingw installations provisioner "powershell" { @@ -178,20 +159,9 @@ build { } ### end of mingw installations - ### vs-2019 installations + # clean unnecessary files provisioner "powershell" { execute_command = var.execute_command - script = "scripts/windows_install_winflexbison.ps1" + script = "scripts/windows_clean_unnecessary_files.ps1" } - - provisioner "powershell" { - execute_command = var.execute_command - script = "scripts/windows_install_pg_deps.ps1" - } - - provisioner "powershell" { - execute_command = var.execute_command - script = "scripts/windows_install_vs_2019.ps1" - } - ### end of vs-2019 installations } diff --git a/packer/windows_base.pkr.hcl b/packer/windows_base.pkr.hcl new file mode 100644 index 0000000..2984edc --- /dev/null +++ b/packer/windows_base.pkr.hcl @@ -0,0 +1,77 @@ +variable "image_name" { type = string } +variable "image_date" { type = string } +variable "gcp_project" {type = string} + +# Packer doesn't capture errors correctly when default execute command is used. +# See $ErrorActionPreference = 'Stop' in the new execute_command. +# So, use new execute_command to handle VM errors correctly +variable "execute_command" { + type = string + default = "powershell -executionpolicy bypass \"& { if (Test-Path variable:global:ProgressPreference){$ProgressPreference='SilentlyContinue'}; $ErrorActionPreference = 'Stop' ;. {{.Vars}}; &'{{.Path}}'; exit $LastExitCode }\"" +} + +locals { + image_identity = "${var.image_name}-${var.image_date}" +} + +source "googlecompute" "windows-base" { + disk_size = "50" + disk_type = "pd-ssd" + project_id = var.gcp_project + source_image_family = "windows-2022" + image_name = local.image_identity + zone = "us-west1-a" + machine_type = "t2d-standard-4" + instance_name = "build-${local.image_identity}" + communicator = "winrm" + winrm_username = "packer_user" + winrm_insecure = true + winrm_use_ssl = true + winrm_timeout = "10m" + state_timeout = "10m" + metadata = { + windows-startup-script-cmd = "winrm quickconfig -quiet & net user /add packer_user & net localgroup administrators packer_user /add & winrm set winrm/config/service/auth @{Basic=\"true\"}" + } +} + +build { + sources = ["googlecompute.windows-base"] + + provisioner "powershell" { + execute_command = var.execute_command + inline = [ + "$ErrorActionPreference = 'Stop'", + # disable antivirus + "Set-MpPreference -DisableRealtimeMonitoring $true -SubmitSamplesConsent NeverSend -MAPSReporting Disable", + + # install choco + "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))", + + "choco install -y --no-progress 7zip", + "choco install -y --no-progress git --parameters=\"/GitAndUnixToolsOnPath\"", + ] + } + + provisioner "powershell" { + execute_command = var.execute_command + inline = [ + "$ErrorActionPreference = 'Stop'", + # contains useful utilities, including a diff we can use + "[Environment]::SetEnvironmentVariable('PATH', 'C:\\Program Files\\Git\\usr\\bin;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine')", + + # set BASE_IMAGE_IDENTITY to distinguish images on CI runs + "[Environment]::SetEnvironmentVariable('BASE_IMAGE_IDENTITY', '${local.image_identity}', 'Machine')", + ] + } + + provisioner "powershell" { + execute_command = var.execute_command + script = "scripts/windows_install_vs_2019.ps1" + } + + # clean unnecessary files + provisioner "powershell" { + execute_command = var.execute_command + script = "scripts/windows_clean_unnecessary_files.ps1" + } +} diff --git a/scripts/windows_clean_unnecessary_files.ps1 b/scripts/windows_clean_unnecessary_files.ps1 new file mode 100644 index 0000000..fa69324 --- /dev/null +++ b/scripts/windows_clean_unnecessary_files.ps1 @@ -0,0 +1,7 @@ +# Do not error out if files do not exist +$ErrorActionPreference = "SilentlyContinue" + +cd c:\ +Remove-Item C:\t -Force -Recurse +Remove-Item -Force -Recurse ${Env:TEMP}\* +Remove-Item -Force -Recurse "${Env:ProgramData}\Package Cache" diff --git a/scripts/windows_install_openssl.ps1 b/scripts/windows_install_openssl.ps1 deleted file mode 100644 index 14404d3..0000000 --- a/scripts/windows_install_openssl.ps1 +++ /dev/null @@ -1,24 +0,0 @@ -# Install openssl -# -# This is one of the windows binaries referenced by the openssl wiki. See -# https://www.openssl.org/community/binaries.html and -# https://wiki.openssl.org/index.php/Binaries -# -# It might be nicer to switch to the openssl built as part of curl-for-win, -# but recent releases only build openssl 3, and that still seems troublesome -# on Windows. - -$ErrorActionPreference = "Stop" - -$filepath = "$Env:TEMP/openssl-setup.exe" - -echo "downloading openssl" -curl.exe -o "$filepath" -fsSL https://slproweb.com/download/Win64OpenSSL-1_1_1w.exe -if (!$?) { throw 'cmdfail' } - -echo "installing openssl" -Start-Process -Wait -FilePath "$filepath" ` - -ArgumentList '/DIR=c:\openssl\1.1\ /VERYSILENT /SP- /SUPPRESSMSGBOXES' -if (!$?) { throw 'cmdfail' } - -Remove-Item "$filepath" -Force diff --git a/scripts/windows_install_pg_deps.ps1 b/scripts/windows_install_pg_deps.ps1 index 1456f80..0cfafb0 100644 --- a/scripts/windows_install_pg_deps.ps1 +++ b/scripts/windows_install_pg_deps.ps1 @@ -1,24 +1,47 @@ $ErrorActionPreference = "Stop" -function DownloadAndInstallDependency($DependencyName, $SourceUri) -{ - echo "downloading $($DependencyName)" - curl.exe -fsSL -o "c:\$($DependencyName).zip" $SourceUri; - if (!$?) { throw 'cmdfail' } - - echo "installing $($DependencyName)" - 7z.exe x "c:\$($DependencyName).zip" -o"c:\$($DependencyName)" - if (!$?) { throw 'cmdfail' } - Remove-Item "c:\$($DependencyName).zip" -Force -} - -DownloadAndInstallDependency "icu" "https://github.com/unicode-org/icu/releases/download/release-71-1/icu4c-71_1-Win64-MSVC2019.zip"; -DownloadAndInstallDependency "lz4" "https://github.com/lz4/lz4/releases/download/v1.9.3/lz4_win64_v1_9_3.zip"; -DownloadAndInstallDependency "zlib" "http://gnuwin32.sourceforge.net/downlinks/zlib-lib-zip.php"; -DownloadAndInstallDependency "zstd" "https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-v1.5.2-win64.zip"; - -[Environment]::SetEnvironmentVariable('PATH', 'C:\winflexbison;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') -[Environment]::SetEnvironmentVariable('PATH', 'C:\icu;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') -[Environment]::SetEnvironmentVariable('PATH', 'C:\lz4;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') -[Environment]::SetEnvironmentVariable('PATH', 'C:\zlib;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') -[Environment]::SetEnvironmentVariable('PATH', 'C:\zstd\zstd-v1.5.2-win64;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') +$VCPKG_PATH = "c:\vcpkg" +$VCPKG_CACHE_PATH = "${VCPKG_PATH}\binary_cache\" +$CIRRUS_BUILD_ID = ${Env:CIRRUS_BUILD_ID} +$VCPKG_PACKAGES = ${Env:VCPKG_PACKAGES} + +vcvarsall.bat x64 +git clone --depth 1 https://github.com/Microsoft/vcpkg.git "${VCPKG_PATH}" +cd "${VCPKG_PATH}" + +mkdir -p binary_cache, downloads +$ARTIFACT_NAME = "vcpkg_cache.zip" +$ARTIFACT_URL = "https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-vcpkg-cache/vcpkg_cache_zip/${ARTIFACT_NAME}" + +echo "Downloading ${ARTIFACT_URL}" +curl.exe -fsSLO ${ARTIFACT_URL} +if (!$?) { throw 'cmdfail' } + +echo "Extracting the cache" +7z.exe x ${ARTIFACT_NAME} -o"${VCPKG_CACHE_PATH}" +if (!$?) { throw 'cmdfail' } + +echo "Installing packages via vcpkg" +.\bootstrap-vcpkg.bat -disableMetrics +.\vcpkg.exe install --debug --clean-after-build --binarysource=files,${VCPKG_CACHE_PATH},readwrite ${VCPKG_PACKAGES} --triplet=x64-windows +if (!$?) { throw 'cmdfail' } + +dir +(du -sh *) -or $true +(find . -name '*.exe' -or -name '*.pc') -or $true + +$VCPKG_PKG_PREFIX = "${VCPKG_PATH}\installed\x64-windows" + +$PKG_PATHS = "${VCPKG_PKG_PREFIX}\debug\lib;" + + "${VCPKG_PKG_PREFIX}\debug\bin;" + + "${VCPKG_PKG_PREFIX}\tools\pkgconf;" + + "${VCPKG_PKG_PREFIX}\tools\python3;" + + "${VCPKG_PKG_PREFIX}\tools\python3\Scripts;" + +# create symbolic link for python3 +$VCPKG_PYTHON_PATH = "${VCPKG_PKG_PREFIX}\tools\python3" +New-Item -ItemType SymbolicLink -Path "${VCPKG_PYTHON_PATH}" -Name "python3.exe" -Value "${VCPKG_PYTHON_PATH}\python.exe" + +[Environment]::SetEnvironmentVariable('PATH', ${PKG_PATHS} + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') +[Environment]::SetEnvironmentVariable('PKG_CONFIG', 'pkgconf', 'Machine') +[Environment]::SetEnvironmentVariable('PKG_CONFIG_PATH', "${VCPKG_PATH}\installed\x64-windows\debug\lib\pkgconfig", 'Machine') diff --git a/scripts/windows_install_python.ps1 b/scripts/windows_install_python.ps1 deleted file mode 100644 index b17781b..0000000 --- a/scripts/windows_install_python.ps1 +++ /dev/null @@ -1,22 +0,0 @@ -$ErrorActionPreference = "Stop" - -$python_version = $Env:TEMP_PYTHON_VERSION -$filepath = "$Env:TEMP/python.exe" - -echo "downloading python $python_version" -curl.exe -fsSL -o "$filepath" https://www.python.org/ftp/python/$python_version/python-$python_version-amd64.exe -if (!$?) { throw 'cmdfail' } - -echo 'installing python' - -Start-Process -Wait -FilePath "$filepath" ` - -ArgumentList ` - '/quiet', 'SimpleInstall=1', 'PrependPath=1', 'CompileAll=1', ` - 'TargetDir=c:\python\', 'InstallAllUsers=1', 'Shortcuts=0', ` - 'Include_docs=0', 'Include_tcltk=0', 'Include_tests=0' -if (!$?) { throw 'cmdfail' } - -# create symbolic link for python3 -New-Item -ItemType SymbolicLink -Path "C:\python" -Name "python3.exe" -Value "C:\python\python.exe" - -Remove-Item "$filepath" -Force diff --git a/scripts/windows_install_vs_2019.ps1 b/scripts/windows_install_vs_2019.ps1 index b1aba0e..71ef99d 100644 --- a/scripts/windows_install_vs_2019.ps1 +++ b/scripts/windows_install_vs_2019.ps1 @@ -31,8 +31,3 @@ rm -r "C:\Program Files (x86)\Windows Kits\10\Lib\*\ucrt\arm64" rm -r "C:\Program Files (x86)\Windows Kits\10\Lib\*\um\arm" rm -r "C:\Program Files (x86)\Windows Kits\10\Lib\*\um\arm64" du -shc "" "C:\Program Files (x86)\Windows Kits" - -cd c:\ -Remove-Item C:\t -Force -Recurse -Remove-Item -Force -Recurse ${Env:TEMP}\* -Remove-Item -Force -Recurse "${Env:ProgramData}\Package Cache" diff --git a/scripts/windows_install_winflexbison.ps1 b/scripts/windows_install_winflexbison.ps1 index 7204a55..315689d 100644 --- a/scripts/windows_install_winflexbison.ps1 +++ b/scripts/windows_install_winflexbison.ps1 @@ -14,4 +14,6 @@ if (!$?) { throw 'cmdfail' } Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe +[Environment]::SetEnvironmentVariable('PATH', 'C:\winflexbison;' + [Environment]::GetEnvironmentVariable('PATH', 'Machine'), 'Machine') + Remove-Item "$filepath" -Force