policy: Run only a single integration test at once. (#8190)

We sometimes see test timeouts where a process's exit code can't be
obtained within 2 minutes(!). This change reduces policy-test
concurrency to 1 to try to avoid resource contention in CI.

This change also fixes duplicate curl exit code logs; and it changes the
default test log level to default to `trace` except for known noisy
dependencies.

Signed-off-by: Oliver Gould <[email protected]>

.github/workflows/policy_controller.yml

@@ -233,5 +233,6 @@ jobs:
       - run: linkerd install | kubectl apply -f -
       - run: linkerd check --wait=1m
 
-      # Run the tests.
-      - run: cargo test -p linkerd-policy-test --frozen
+      # Run the tests. We disable parallelism to avoid spurious timeouts caused
+      # by resource contention.
+      - run: cargo test -p linkerd-policy-test --frozen -- --test-threads=1

policy-test/src/curl.rs

@@ -1,5 +1,4 @@
 use super::{create, LinkerdInject};
-use kube::ResourceExt;
 use linkerd_policy_controller_k8s_api::{self as k8s};
 use maplit::{btreemap, convert_args};
 use tokio::time;
@@ -208,7 +207,6 @@ impl Running {
                 .iter()
                 .find(|c| c.name == "curl")?;
             let code = c.state.as_ref()?.terminated.as_ref()?.exit_code;
-            tracing::debug!(ns = %pod.namespace().unwrap(), pod = %pod.name(), %code, "Curl exited");
             Some(code)
         }
 
@@ -222,19 +220,22 @@ impl Running {
         match time::timeout(time::Duration::from_secs(120), finished).await {
             Ok(Ok(())) => {}
             Ok(Err(error)) => panic!("Failed to wait for exit code: {}: {}", self.name, error),
-            Err(_timeout) => panic!("Timeout waiting for exit code: {}", self.name),
+            Err(_timeout) => {
+                panic!("Timeout waiting for exit code: {}", self.name);
+            }
         };
 
         let curl_pod = api.get(&self.name).await.expect("pod must exist");
-        let ex = get_exit_code(&curl_pod).expect("curl pod must have an exit code");
+        let code = get_exit_code(&curl_pod).expect("curl pod must have an exit code");
+        tracing::debug!(pod = %self.name, %code, "Curl exited");
 
         if let Err(error) = api
-            .delete(&self.name, &kube::api::DeleteParams::background())
+            .delete(&self.name, &kube::api::DeleteParams::foreground())
             .await
         {
             tracing::trace!(%error, name = %self.name, "Failed to delete pod");
         }
 
-        ex
+        code
     }
 }

policy-test/src/lib.rs

@@ -133,7 +133,23 @@ where
     let test = test(client.clone(), ns.name());
     let res = tokio::spawn(test.instrument(tracing::info_span!("test", ns = %ns.name()))).await;
     if res.is_err() {
-        // If the test failed, stop tracing so the log is not polluted with more
+        // If the test failed, list the state of all pods/containers in the namespace.
+        let pods = kube::Api::<k8s::Pod>::namespaced(client.clone(), &ns.name())
+            .list(&Default::default())
+            .await
+            .expect("Failed to get pod status");
+        for p in pods.items {
+            let pod = p.name();
+            if let Some(status) = p.status {
+                let _span = tracing::info_span!("pod", ns = %ns.name(), name = %pod).entered();
+                tracing::trace!(reason = ?status.reason, message = ?status.message);
+                for c in status.container_statuses.into_iter().flatten() {
+                    tracing::trace!(container = %c.name, ready = %c.ready, state = ?c.state);
+                }
+            }
+        }
+
+        // Then stop tracing so the log is not further polluted with more
         // information about cleanup after the failure was printed.
         drop(_tracing);
     }
@@ -163,7 +179,7 @@ fn init_tracing() -> tracing::subscriber::DefaultGuard {
             .with_test_writer()
             .with_env_filter(
                 tracing_subscriber::EnvFilter::try_from_default_env()
-                    .unwrap_or_else(|_| "e2e=trace,api=trace,linkerd=trace,info".parse().unwrap()),
+                    .unwrap_or_else(|_| "trace,hyper=info,kube=info,h2=info".parse().unwrap()),
             )
             .finish(),
     )