From 5907b3b32899c8134605dc8b797f4d1cadbff039 Mon Sep 17 00:00:00 2001
From: Praveen Kumar <kumarpraveen.nitdgp@gmail.com>
Date: Thu, 1 Feb 2024 13:22:28 +0530
Subject: [PATCH] Check status of all the core pods for microshift

In past we observed having kube api access doesn't mean all the required
service pods are running and cluster is working as expected. This PR
adds a list of core namespace for microshift preset and make sure all
the pods in that namespace is running before letting user to know to
consume the cluster.
---
 pkg/crc/cluster/cluster.go | 26 ++++++++++++++++++++++++++
 pkg/crc/machine/start.go   |  5 ++++-
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/pkg/crc/cluster/cluster.go b/pkg/crc/cluster/cluster.go
index ad2201358a..990ae73c20 100644
--- a/pkg/crc/cluster/cluster.go
+++ b/pkg/crc/cluster/cluster.go
@@ -510,3 +510,29 @@ func DeleteMCOLeaderLease(ctx context.Context, ocConfig oc.Config) error {
 	_, _, err := ocConfig.RunOcCommand("delete", "-A", "lease", "--all")
 	return err
 }
+
+func CheckCorePodsRunning(ctx context.Context, ocConfig oc.Config) error {
+	coreNameSpace := []string{"kube-system", "openshift-dns", "openshift-ingress", "openshift-ovn-kubernetes", "openshift-service-ca"}
+	waitForPods := func() error {
+		for _, namespace := range coreNameSpace {
+			if !podRunningForNamespace(ocConfig, namespace) {
+				logging.Debugf("Pods in %s namespace are not running", namespace)
+				return &errors.RetriableError{Err: fmt.Errorf("pods in %s namespace are not running", namespace)}
+			}
+		}
+		return nil
+	}
+	return errors.Retry(ctx, 2*time.Minute, waitForPods, 2*time.Second)
+}
+
+func podRunningForNamespace(ocConfig oc.Config, namespace string) bool {
+	stdin, stderr, err := ocConfig.WithFailFast().RunOcCommand("get", "pods", "-n", namespace, "--field-selector=status.phase!=Running")
+	if err != nil {
+		logging.Debugf("Failed to get pods in %s namespace, stderr: %s", namespace, stderr)
+		return false
+	}
+	if len(stdin) != 0 {
+		return false
+	}
+	return true
+}
diff --git a/pkg/crc/machine/start.go b/pkg/crc/machine/start.go
index 967f0b2512..d349e9ed6f 100644
--- a/pkg/crc/machine/start.go
+++ b/pkg/crc/machine/start.go
@@ -1064,7 +1064,10 @@ func startMicroshift(ctx context.Context, sshRunner *crcssh.Runner, ocConfig oc.
 		return err
 	}
 
-	return cluster.WaitForAPIServer(ctx, ocConfig)
+	if err := cluster.WaitForAPIServer(ctx, ocConfig); err != nil {
+		return err
+	}
+	return cluster.CheckCorePodsRunning(ctx, ocConfig)
 }
 
 func ensurePullSecretPresentInVM(sshRunner *crcssh.Runner, pullSec cluster.PullSecretLoader) error {