From dbd66a7f81dd4ce6a66c49ae7f27c2dc6c0b8bc1 Mon Sep 17 00:00:00 2001 From: nginx-bot Date: Fri, 3 Jan 2025 22:14:48 +0000 Subject: [PATCH] NFR Test Results for NGF version edge --- tests/results/reconfig/edge/edge-oss.md | 194 ++++++++++---------- tests/results/reconfig/edge/edge-plus.md | 215 ++++++++++++----------- 2 files changed, 223 insertions(+), 186 deletions(-) diff --git a/tests/results/reconfig/edge/edge-oss.md b/tests/results/reconfig/edge/edge-oss.md index 98bbb75a07..268350f6b9 100644 --- a/tests/results/reconfig/edge/edge-oss.md +++ b/tests/results/reconfig/edge/edge-oss.md @@ -6,16 +6,16 @@ NGINX Plus: false NGINX Gateway Fabric: -- Commit: e7d217a8f01fb3c8fc4507ef6f0e7feead667f20 -- Date: 2024-11-14T18:42:55Z +- Commit: 9a47b6d33e6c5e0568af0bfc367e5f1190354b05 +- Date: 2025-01-03T19:08:47Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.30.5-gke.1443001 +- k8s version: v1.30.6-gke.1125000 - vCPUs per node: 16 -- RAM per node: 65853972Ki +- RAM per node: 65853984Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 @@ -27,41 +27,44 @@ GKE Cluster: - TimeToReadyTotal: 3s - TimeToReadyAvgSingle: < 1s - NGINX Reloads: 2 -- NGINX Reload Average Time: 126ms +- NGINX Reload Average Time: 127ms - Reload distribution: - - 500ms: 2 - - 1000ms: 2 - - 5000ms: 2 - - 10000ms: 2 - - 30000ms: 2 + - 500.0ms: 2 + - 1000.0ms: 2 + - 5000.0ms: 2 + - 10000.0ms: 2 + - 30000.0ms: 2 - +Infms: 2 ### Event Batch Processing - Event Batch Total: 5 -- Event Batch Processing Average Time: 60ms +- Event Batch Processing Average Time: 59ms - Event Batch Processing distribution: - - 500ms: 5 - - 1000ms: 5 - - 5000ms: 5 - - 10000ms: 5 - - 30000ms: 5 + - 500.0ms: 5 + - 1000.0ms: 5 + - 5000.0ms: 5 + - 10000.0ms: 5 + - 30000.0ms: 5 - +Infms: 5 +### NGINX Error Logs + + ## Test 1: Resources exist before startup - NumResources 150 ### Reloads and Time to Ready -- TimeToReadyTotal: 2s +- TimeToReadyTotal: 3s - TimeToReadyAvgSingle: < 1s - NGINX Reloads: 2 - NGINX Reload Average Time: 126ms - Reload distribution: - - 500ms: 2 - - 1000ms: 2 - - 5000ms: 2 - - 10000ms: 2 - - 30000ms: 2 + - 500.0ms: 2 + - 1000.0ms: 2 + - 5000.0ms: 2 + - 10000.0ms: 2 + - 30000.0ms: 2 - +Infms: 2 ### Event Batch Processing @@ -69,40 +72,46 @@ GKE Cluster: - Event Batch Total: 6 - Event Batch Processing Average Time: 50ms - Event Batch Processing distribution: - - 500ms: 6 - - 1000ms: 6 - - 5000ms: 6 - - 10000ms: 6 - - 30000ms: 6 + - 500.0ms: 6 + - 1000.0ms: 6 + - 5000.0ms: 6 + - 10000.0ms: 6 + - 30000.0ms: 6 - +Infms: 6 +### NGINX Error Logs + + ## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 ### Reloads and Time to Ready - TimeToReadyTotal: 8s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 52 -- NGINX Reload Average Time: 151ms +- NGINX Reloads: 53 +- NGINX Reload Average Time: 150ms - Reload distribution: - - 500ms: 52 - - 1000ms: 52 - - 5000ms: 52 - - 10000ms: 52 - - 30000ms: 52 - - +Infms: 52 + - 500.0ms: 53 + - 1000.0ms: 53 + - 5000.0ms: 53 + - 10000.0ms: 53 + - 30000.0ms: 53 + - +Infms: 53 ### Event Batch Processing -- Event Batch Total: 326 +- Event Batch Total: 329 - Event Batch Processing Average Time: 24ms - Event Batch Processing distribution: - - 500ms: 326 - - 1000ms: 326 - - 5000ms: 326 - - 10000ms: 326 - - 30000ms: 326 - - +Infms: 326 + - 500.0ms: 329 + - 1000.0ms: 329 + - 5000.0ms: 329 + - 10000.0ms: 329 + - 30000.0ms: 329 + - +Infms: 329 + +### NGINX Error Logs + ## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 @@ -110,27 +119,30 @@ GKE Cluster: - TimeToReadyTotal: 43s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 283 -- NGINX Reload Average Time: 152ms +- NGINX Reloads: 287 +- NGINX Reload Average Time: 151ms - Reload distribution: - - 500ms: 283 - - 1000ms: 283 - - 5000ms: 283 - - 10000ms: 283 - - 30000ms: 283 - - +Infms: 283 + - 500.0ms: 287 + - 1000.0ms: 287 + - 5000.0ms: 287 + - 10000.0ms: 287 + - 30000.0ms: 287 + - +Infms: 287 ### Event Batch Processing -- Event Batch Total: 1633 +- Event Batch Total: 1640 - Event Batch Processing Average Time: 26ms - Event Batch Processing distribution: - - 500ms: 1633 - - 1000ms: 1633 - - 5000ms: 1633 - - 10000ms: 1633 - - 30000ms: 1633 - - +Infms: 1633 + - 500.0ms: 1640 + - 1000.0ms: 1640 + - 5000.0ms: 1640 + - 10000.0ms: 1640 + - 30000.0ms: 1640 + - +Infms: 1640 + +### NGINX Error Logs + ## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 @@ -138,27 +150,30 @@ GKE Cluster: - TimeToReadyTotal: < 1s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 55 -- NGINX Reload Average Time: 148ms +- NGINX Reloads: 58 +- NGINX Reload Average Time: 139ms - Reload distribution: - - 500ms: 55 - - 1000ms: 55 - - 5000ms: 55 - - 10000ms: 55 - - 30000ms: 55 - - +Infms: 55 + - 500.0ms: 58 + - 1000.0ms: 58 + - 5000.0ms: 58 + - 10000.0ms: 58 + - 30000.0ms: 58 + - +Infms: 58 ### Event Batch Processing -- Event Batch Total: 296 +- Event Batch Total: 300 - Event Batch Processing Average Time: 27ms - Event Batch Processing distribution: - - 500ms: 296 - - 1000ms: 296 - - 5000ms: 296 - - 10000ms: 296 - - 30000ms: 296 - - +Infms: 296 + - 500.0ms: 300 + - 1000.0ms: 300 + - 5000.0ms: 300 + - 10000.0ms: 300 + - 30000.0ms: 300 + - +Infms: 300 + +### NGINX Error Logs + ## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 @@ -166,24 +181,27 @@ GKE Cluster: - TimeToReadyTotal: < 1s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 294 -- NGINX Reload Average Time: 148ms +- NGINX Reloads: 324 +- NGINX Reload Average Time: 134ms - Reload distribution: - - 500ms: 294 - - 1000ms: 294 - - 5000ms: 294 - - 10000ms: 294 - - 30000ms: 294 - - +Infms: 294 + - 500.0ms: 324 + - 1000.0ms: 324 + - 5000.0ms: 324 + - 10000.0ms: 324 + - 30000.0ms: 324 + - +Infms: 324 ### Event Batch Processing -- Event Batch Total: 1506 -- Event Batch Processing Average Time: 29ms +- Event Batch Total: 1520 +- Event Batch Processing Average Time: 28ms - Event Batch Processing distribution: - - 500ms: 1506 - - 1000ms: 1506 - - 5000ms: 1506 - - 10000ms: 1506 - - 30000ms: 1506 - - +Infms: 1506 + - 500.0ms: 1520 + - 1000.0ms: 1520 + - 5000.0ms: 1520 + - 10000.0ms: 1520 + - 30000.0ms: 1520 + - +Infms: 1520 + +### NGINX Error Logs + diff --git a/tests/results/reconfig/edge/edge-plus.md b/tests/results/reconfig/edge/edge-plus.md index b03b8cc83f..56d7e12db9 100644 --- a/tests/results/reconfig/edge/edge-plus.md +++ b/tests/results/reconfig/edge/edge-plus.md @@ -6,16 +6,16 @@ NGINX Plus: true NGINX Gateway Fabric: -- Commit: e7d217a8f01fb3c8fc4507ef6f0e7feead667f20 -- Date: 2024-11-14T18:42:55Z +- Commit: 9a47b6d33e6c5e0568af0bfc367e5f1190354b05 +- Date: 2025-01-03T19:08:47Z - Dirty: false GKE Cluster: - Node count: 12 -- k8s version: v1.30.5-gke.1443001 +- k8s version: v1.30.6-gke.1125000 - vCPUs per node: 16 -- RAM per node: 65853972Ki +- RAM per node: 65853984Ki - Max pods per node: 110 - Zone: us-west1-b - Instance Type: n2d-standard-16 @@ -24,85 +24,94 @@ GKE Cluster: ### Reloads and Time to Ready -- TimeToReadyTotal: 3s +- TimeToReadyTotal: 5s - TimeToReadyAvgSingle: < 1s - NGINX Reloads: 2 -- NGINX Reload Average Time: 114ms +- NGINX Reload Average Time: 125ms - Reload distribution: - - 500ms: 2 - - 1000ms: 2 - - 5000ms: 2 - - 10000ms: 2 - - 30000ms: 2 + - 500.0ms: 2 + - 1000.0ms: 2 + - 5000.0ms: 2 + - 10000.0ms: 2 + - 30000.0ms: 2 - +Infms: 2 ### Event Batch Processing -- Event Batch Total: 5 -- Event Batch Processing Average Time: 56ms +- Event Batch Total: 6 +- Event Batch Processing Average Time: 57ms - Event Batch Processing distribution: - - 500ms: 5 - - 1000ms: 5 - - 5000ms: 5 - - 10000ms: 5 - - 30000ms: 5 - - +Infms: 5 + - 500.0ms: 6 + - 1000.0ms: 6 + - 5000.0ms: 6 + - 10000.0ms: 6 + - 30000.0ms: 6 + - +Infms: 6 + +### NGINX Error Logs + ## Test 1: Resources exist before startup - NumResources 150 ### Reloads and Time to Ready -- TimeToReadyTotal: 1s +- TimeToReadyTotal: 5s - TimeToReadyAvgSingle: < 1s - NGINX Reloads: 2 -- NGINX Reload Average Time: 113ms +- NGINX Reload Average Time: 139ms - Reload distribution: - - 500ms: 2 - - 1000ms: 2 - - 5000ms: 2 - - 10000ms: 2 - - 30000ms: 2 + - 500.0ms: 2 + - 1000.0ms: 2 + - 5000.0ms: 2 + - 10000.0ms: 2 + - 30000.0ms: 2 - +Infms: 2 ### Event Batch Processing - Event Batch Total: 6 -- Event Batch Processing Average Time: 46ms +- Event Batch Processing Average Time: 65ms - Event Batch Processing distribution: - - 500ms: 6 - - 1000ms: 6 - - 5000ms: 6 - - 10000ms: 6 - - 30000ms: 6 + - 500.0ms: 6 + - 1000.0ms: 6 + - 5000.0ms: 6 + - 10000.0ms: 6 + - 30000.0ms: 6 - +Infms: 6 +### NGINX Error Logs + + ## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 30 ### Reloads and Time to Ready -- TimeToReadyTotal: 8s +- TimeToReadyTotal: 7s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 62 -- NGINX Reload Average Time: 125ms +- NGINX Reloads: 45 +- NGINX Reload Average Time: 155ms - Reload distribution: - - 500ms: 62 - - 1000ms: 62 - - 5000ms: 62 - - 10000ms: 62 - - 30000ms: 62 - - +Infms: 62 + - 500.0ms: 45 + - 1000.0ms: 45 + - 5000.0ms: 45 + - 10000.0ms: 45 + - 30000.0ms: 45 + - +Infms: 45 ### Event Batch Processing -- Event Batch Total: 337 -- Event Batch Processing Average Time: 23ms +- Event Batch Total: 321 +- Event Batch Processing Average Time: 25ms - Event Batch Processing distribution: - - 500ms: 337 - - 1000ms: 337 - - 5000ms: 337 - - 10000ms: 337 - - 30000ms: 337 - - +Infms: 337 + - 500.0ms: 321 + - 1000.0ms: 321 + - 5000.0ms: 321 + - 10000.0ms: 321 + - 30000.0ms: 321 + - +Infms: 321 + +### NGINX Error Logs + ## Test 2: Start NGF, deploy Gateway, create many resources attached to GW - NumResources 150 @@ -110,55 +119,62 @@ GKE Cluster: - TimeToReadyTotal: 43s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 338 -- NGINX Reload Average Time: 126ms +- NGINX Reloads: 245 +- NGINX Reload Average Time: 152ms - Reload distribution: - - 500ms: 338 - - 1000ms: 338 - - 5000ms: 338 - - 10000ms: 338 - - 30000ms: 338 - - +Infms: 338 + - 500.0ms: 245 + - 1000.0ms: 245 + - 5000.0ms: 245 + - 10000.0ms: 245 + - 30000.0ms: 245 + - +Infms: 245 ### Event Batch Processing -- Event Batch Total: 1693 -- Event Batch Processing Average Time: 25ms +- Event Batch Total: 1600 +- Event Batch Processing Average Time: 27ms - Event Batch Processing distribution: - - 500ms: 1693 - - 1000ms: 1693 - - 5000ms: 1693 - - 10000ms: 1693 - - 30000ms: 1693 - - +Infms: 1693 + - 500.0ms: 1600 + - 1000.0ms: 1600 + - 5000.0ms: 1600 + - 10000.0ms: 1600 + - 30000.0ms: 1600 + - +Infms: 1600 + +### NGINX Error Logs + ## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 30 ### Reloads and Time to Ready -- TimeToReadyTotal: < 1s +- TimeToReadyTotal: -4s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 64 -- NGINX Reload Average Time: 125ms +- NGINX Reloads: 25 +- NGINX Reload Average Time: 149ms - Reload distribution: - - 500ms: 64 - - 1000ms: 64 - - 5000ms: 64 - - 10000ms: 64 - - 30000ms: 64 - - +Infms: 64 + - 500.0ms: 25 + - 1000.0ms: 25 + - 5000.0ms: 25 + - 10000.0ms: 25 + - 30000.0ms: 25 + - +Infms: 25 ### Event Batch Processing -- Event Batch Total: 306 -- Event Batch Processing Average Time: 26ms +- Event Batch Total: 266 +- Event Batch Processing Average Time: 18ms - Event Batch Processing distribution: - - 500ms: 306 - - 1000ms: 306 - - 5000ms: 306 - - 10000ms: 306 - - 30000ms: 306 - - +Infms: 306 + - 500.0ms: 266 + - 1000.0ms: 266 + - 5000.0ms: 266 + - 10000.0ms: 266 + - 30000.0ms: 266 + - +Infms: 266 + +### NGINX Error Logs +2025/01/03 21:27:53 [emerg] 45#45: invalid instance state file "/var/lib/nginx/state/nginx-mgmt-state" + ## Test 3: Start NGF, create many resources attached to a Gateway, deploy the Gateway - NumResources 150 @@ -166,24 +182,27 @@ GKE Cluster: - TimeToReadyTotal: < 1s - TimeToReadyAvgSingle: < 1s -- NGINX Reloads: 342 -- NGINX Reload Average Time: 125ms +- NGINX Reloads: 231 +- NGINX Reload Average Time: 151ms - Reload distribution: - - 500ms: 342 - - 1000ms: 342 - - 5000ms: 342 - - 10000ms: 342 - - 30000ms: 342 - - +Infms: 342 + - 500.0ms: 231 + - 1000.0ms: 231 + - 5000.0ms: 231 + - 10000.0ms: 231 + - 30000.0ms: 231 + - +Infms: 231 ### Event Batch Processing -- Event Batch Total: 1534 -- Event Batch Processing Average Time: 28ms +- Event Batch Total: 1447 +- Event Batch Processing Average Time: 30ms - Event Batch Processing distribution: - - 500ms: 1534 - - 1000ms: 1534 - - 5000ms: 1534 - - 10000ms: 1534 - - 30000ms: 1534 - - +Infms: 1534 + - 500.0ms: 1446 + - 1000.0ms: 1447 + - 5000.0ms: 1447 + - 10000.0ms: 1447 + - 30000.0ms: 1447 + - +Infms: 1447 + +### NGINX Error Logs +