From 2d9330ac3cbf6f683cb6ea531b499945d3f83b2f Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Wed, 25 Oct 2023 17:10:38 -0700 Subject: [PATCH 1/5] create kind network Signed-off-by: Manabu Mccloskey --- pkg/controllers/gitserver/image_test.go | 51 ++++++++++++++++++++++--- pkg/kind/registry_test.go | 8 ++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/gitserver/image_test.go b/pkg/controllers/gitserver/image_test.go index 9efcbd21..7741610c 100644 --- a/pkg/controllers/gitserver/image_test.go +++ b/pkg/controllers/gitserver/image_test.go @@ -3,6 +3,11 @@ package gitserver import ( "context" "fmt" + "github.com/cnoe-io/idpbuilder/pkg/kind" + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "io" + "os" "strings" "testing" @@ -39,20 +44,54 @@ func TestReconcileGitServerImage(t *testing.T) { }, } - _, err = r.reconcileGitServerImage(ctx, controllerruntime.Request{}, &resource) + dockerClient, err := docker.GetDockerClient() + if err != nil { + t.Errorf("Getting docker client: %v", err) + } + defer dockerClient.Close() + reader, err := dockerClient.ImagePull(ctx, "docker.io/library/registry:2", types.ImagePullOptions{}) + defer reader.Close() + // blocks until pull is completed + io.Copy(os.Stdout, reader) if err != nil { - t.Errorf("reconcile error: %v", err) + t.Fatalf("failed pulilng registry image: %v", err) } - if !strings.HasPrefix(resource.Status.ImageID, "sha256") { - t.Errorf("Invalid or no Image ID in status: %q", resource.Status.ImageID) + resp, err := dockerClient.ContainerCreate(ctx, &container.Config{ + Image: "docker.io/library/registry:2", + Tty: false, + ExposedPorts: nat.PortSet{ + nat.Port(fmt.Sprintf("%d/tcp", kind.InternalRegistryPort)): struct{}{}, + }, + }, &container.HostConfig{ + PortBindings: nat.PortMap{ + nat.Port(fmt.Sprintf("%d/tcp", kind.InternalRegistryPort)): []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: fmt.Sprintf("%d", kind.ExposedRegistryPort), + }, + }, + }, + }, nil, nil, "testcase-registry") + if err != nil { + t.Fatalf("failed creating registry container %v", err) } - dockerClient, err := docker.GetDockerClient() + defer dockerClient.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) + + err = dockerClient.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) if err != nil { - t.Errorf("Getting docker client: %v", err) + t.Fatalf("failed starting container %v", err) } + _, err = r.reconcileGitServerImage(ctx, controllerruntime.Request{}, &resource) + if err != nil { + t.Fatalf("reconcile error: %v", err) + } + + if !strings.HasPrefix(resource.Status.ImageID, "sha256") { + t.Fatalf("Invalid or no Image ID in status: %q", resource.Status.ImageID) + } imageNameID := fmt.Sprintf("%s@%s", GetImageTag(&resource), resource.Status.ImageID) _, err = dockerClient.ImageRemove(ctx, imageNameID, types.ImageRemoveOptions{}) if err != nil { diff --git a/pkg/kind/registry_test.go b/pkg/kind/registry_test.go index 06309b8b..059cefca 100644 --- a/pkg/kind/registry_test.go +++ b/pkg/kind/registry_test.go @@ -19,6 +19,12 @@ func TestReconcileRegistry(t *testing.T) { } defer dockerCli.Close() + kindNetwork, err := dockerCli.NetworkCreate(ctx, "kind", types.NetworkCreate{}) + if err != nil { + t.Fatalf("Failed creaking kind network: %v", err) + } + defer dockerCli.NetworkRemove(ctx, kindNetwork.ID) + // Create cluster cluster, err := NewCluster("testcase", "v1.26.3", "", "", "") if err != nil { @@ -36,6 +42,8 @@ func TestReconcileRegistry(t *testing.T) { if err != nil { t.Fatalf("Error getting registry container after reconcile: %v", err) } + defer dockerCli.ImageRemove(ctx, container.ImageID, types.ImageRemoveOptions{}) + if container == nil { t.Fatal("Expected registry container after reconcile but got nil") } From a79b6ec8665de7648ed6771248e1dfa49f581087 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Thu, 26 Oct 2023 15:28:40 -0700 Subject: [PATCH 2/5] ensure container is removed Signed-off-by: Manabu Mccloskey --- pkg/kind/registry_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/kind/registry_test.go b/pkg/kind/registry_test.go index 059cefca..59b1a9e3 100644 --- a/pkg/kind/registry_test.go +++ b/pkg/kind/registry_test.go @@ -33,6 +33,7 @@ func TestReconcileRegistry(t *testing.T) { // Create registry err = cluster.ReconcileRegistry(ctx) + defer dockerCli.ContainerRemove(ctx, cluster.getRegistryContainerName(), types.ContainerRemoveOptions{Force: true}) if err != nil { t.Fatalf("Error reconciling registry: %v", err) } @@ -53,9 +54,4 @@ func TestReconcileRegistry(t *testing.T) { if err != nil { t.Fatalf("Error reconciling registry: %v", err) } - - // Cleanup - if err = dockerCli.ContainerRemove(ctx, container.ID, types.ContainerRemoveOptions{Force: true}); err != nil { - t.Fatalf("Error removing registry docker container after reconcile: %v", err) - } } From c14e76731b2e248cfa6361888e422551834394a3 Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Thu, 26 Oct 2023 15:54:21 -0700 Subject: [PATCH 3/5] add tcp connection check Signed-off-by: Manabu Mccloskey --- pkg/controllers/gitserver/image_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/controllers/gitserver/image_test.go b/pkg/controllers/gitserver/image_test.go index 7741610c..7a77d2b4 100644 --- a/pkg/controllers/gitserver/image_test.go +++ b/pkg/controllers/gitserver/image_test.go @@ -7,9 +7,12 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "io" + "net" "os" + "strconv" "strings" "testing" + "time" "github.com/cnoe-io/idpbuilder/api/v1alpha1" "github.com/cnoe-io/idpbuilder/pkg/apps" @@ -57,6 +60,22 @@ func TestReconcileGitServerImage(t *testing.T) { t.Fatalf("failed pulilng registry image: %v", err) } + waitTimeout := time.Second * 90 + waitInterval := time.Second * 3 + // very crude. no guarantee that the port will be available by the time request is sent to docker + endTime := time.Now().Add(waitTimeout) + for { + if time.Now().After(endTime) { + t.Fatalf("Timed out waiting for port %d to be available", kind.ExposedRegistryPort) + } + conn, cErr := net.DialTimeout("tcp", net.JoinHostPort("0.0.0.0", strconv.Itoa(int(kind.ExposedRegistryPort))), time.Second*3) + if cErr != nil { + break + } + conn.Close() + time.Sleep(waitInterval) + } + resp, err := dockerClient.ContainerCreate(ctx, &container.Config{ Image: "docker.io/library/registry:2", Tty: false, @@ -98,3 +117,4 @@ func TestReconcileGitServerImage(t *testing.T) { t.Errorf("Removing docker image: %v", err) } } + From 2ff688cc3de43eefa0c4cff27211a0c9d0a2f96a Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Thu, 26 Oct 2023 16:14:12 -0700 Subject: [PATCH 4/5] add retry Signed-off-by: Manabu Mccloskey --- pkg/kind/registry_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/kind/registry_test.go b/pkg/kind/registry_test.go index 59b1a9e3..9d5b6143 100644 --- a/pkg/kind/registry_test.go +++ b/pkg/kind/registry_test.go @@ -3,6 +3,7 @@ package kind import ( "context" "testing" + "time" "github.com/cnoe-io/idpbuilder/pkg/docker" "github.com/docker/docker/api/types" @@ -32,10 +33,21 @@ func TestReconcileRegistry(t *testing.T) { } // Create registry - err = cluster.ReconcileRegistry(ctx) defer dockerCli.ContainerRemove(ctx, cluster.getRegistryContainerName(), types.ContainerRemoveOptions{Force: true}) - if err != nil { - t.Fatalf("Error reconciling registry: %v", err) + waitTimeout := time.Second * 90 + waitInterval := time.Second * 3 + endTime := time.Now().Add(waitTimeout) + + for { + if time.Now().After(endTime) { + t.Fatalf("Timed out waiting for registry. recent error: %v", err) + } + err = cluster.ReconcileRegistry(ctx) + if err == nil { + break + } + t.Logf("Failed to reconcile: %v", err) + time.Sleep(waitInterval) } // Get resulting container @@ -55,3 +67,4 @@ func TestReconcileRegistry(t *testing.T) { t.Fatalf("Error reconciling registry: %v", err) } } + From 4bc985434fa9ab7d6ae4230fd4749f95cceae66c Mon Sep 17 00:00:00 2001 From: Manabu Mccloskey Date: Thu, 26 Oct 2023 16:37:06 -0700 Subject: [PATCH 5/5] remove container after failure Signed-off-by: Manabu Mccloskey --- pkg/kind/registry_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kind/registry_test.go b/pkg/kind/registry_test.go index 9d5b6143..084f8536 100644 --- a/pkg/kind/registry_test.go +++ b/pkg/kind/registry_test.go @@ -47,6 +47,7 @@ func TestReconcileRegistry(t *testing.T) { break } t.Logf("Failed to reconcile: %v", err) + dockerCli.ContainerRemove(ctx, cluster.getRegistryContainerName(), types.ContainerRemoveOptions{Force: true}) time.Sleep(waitInterval) }