Skip to content

Commit

Permalink
chg: test: Add stress tests with DoH and DoT
Browse files Browse the repository at this point in the history 
Validation pipeline: https://gitlab.isc.org/isc-projects/bind9/-/pipelines/160984

Prerequisites:
- [x] isc-private/devops!11
- [x] https://gitlab.isc.org/isc-projects/bind9-qa/-/merge_requests/9

Things to consider:
- FreeBSD DoH jobs are not added because Flamethrower queries always timeout.
- This adds 15 more CI jobs:
  - Linux (AWS autoscaler): `(auth + recursive + RPZ) * (DoH + DoT) * (amd64 + arm64) = 12`
  - FreeBSD (one FreeBSD runner): `(auth + recursive + RPZ) * (DoT) * (amd64) = 3`
- Autoscaler is not yet present on FreeBSD. Adding 3 CI jobs (i.e., DoT) run serially adds 3 hours to the pipeline runtime. Should we add just one FreeBSD DoT job to limit the runtime?
- DoH/DoT performance is slightly lower than pure TCP, so the threshold for the test to pass must be lowered by 5-10% (see isc-private/bind-qa!40).

Merge branch 'mnowak/stress-test-with-doh-dot' into 'main'

See merge request isc-projects/bind9!5800
  • Loading branch information
@Mno-hime
Mno-hime committed Jan 27, 2025
2 parents dc3c3ef + 9756292 commit 92a3487
Show file tree
Hide file tree
Showing 2 changed files with 158 additions and 273 deletions.
292 changes: 19 additions & 273 deletions .gitlab-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,6 @@ variables:
# cross-testrun files as there is no need to use that feature in CI.
PYTEST_ADDOPTS: "-p no:cacheprovider"

# Default platforms to run "stress" tests on
BIND_STRESS_TEST_OS: linux
BIND_STRESS_TEST_ARCH: amd64

HYPOTHESIS_PROFILE: "ci"

default:
Expand Down Expand Up @@ -1699,278 +1695,28 @@ shotgun:doh-get:

.stress-test: &stress_test
stage: performance
script:
- *configure
- *setup_interfaces
- make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
- make DESTDIR="${INSTALL_PATH}" install
- git clone --depth 1 https://gitlab.isc.org/isc-projects/bind9-qa.git
- cd bind9-qa/stress
- LD_LIBRARY_PATH="${INSTALL_PATH}/usr/local/lib" BIND_INSTALL_PATH="${INSTALL_PATH}/usr/local" WORKSPACE="${CI_PROJECT_DIR}" bash stress.sh
needs:
- job: autoreconf
artifacts: true

.stress-test-long: &stress_test_long_job
<<: *stress_test
artifacts:
untracked: true
exclude:
- "output/ns4/*.dtq*"
- "output/ns4/large-delta-rpz*.local"
- "output/rpz_*"
expire_in: "1 week"
when: always
timeout: 2h

.stress-test-short: &stress_test_short_job
<<: *stress_test
only:
- merge_requests
generate-stress-test-configs:
<<: *base_image
<<: *default_triggering_rules
stage: precheck
script:
- util/generate-stress-test-configs.py > stress-test-configs.yml
artifacts:
untracked: true
exclude:
- "output/ns4/*.dtq*"
- "output/ns4/large-delta-rpz*.local"
- "output/rpz_*"
when: always

stress:short:authoritative:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 15

stress:short:recursive:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 15

stress:short:rpz:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 15

stress:short:authoritative:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 15

stress:short:recursive:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 15

stress:short:rpz:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_short_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 15

stress:short:authoritative:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_short_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 15

stress:short:recursive:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_short_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 15

stress:short:rpz:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_short_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 15

stress:authoritative:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)

stress:recursive:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)

stress:rpz:fedora:41:amd64:
<<: *fedora_41_amd64_image
<<: *linux_amd64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)

stress:authoritative:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i)

stress:recursive:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i)

stress:rpz:fedora:41:arm64:
<<: *fedora_41_arm64_image
<<: *linux_arm64
<<: *stress_test_long_job
variables:
CC: gcc
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i)

stress:authoritative:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_long_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: authoritative
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)

stress:recursive:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_long_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: recursive
RATE: 10000
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)
paths:
- stress-test-configs.yml
needs: []

stress:rpz:freebsd13:amd64:
<<: *freebsd_stress_amd64
<<: *stress_test_long_job
variables:
CC: clang
CFLAGS: "${CFLAGS_COMMON} -Og"
FLAME: /usr/local/bin/flame
MODE: rpz
RATE: 1500
RUN_TIME: 60
only:
variables:
- $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i)
stress-test-child-pipeline:
<<: *default_triggering_rules
stage: performance
trigger:
include:
- artifact: stress-test-configs.yml
job: generate-stress-test-configs
needs:
- job: generate-stress-test-configs
artifacts: true

# git fsck operates over the whole repository and is sufficient to schedule it
# only in one branch, preferably "main". GitLab's clone strategy prevents us
Expand Down
Loading

0 comments on commit 92a3487

Please sign in to comment.