Merge pull request kubernetes#21617 from luxas/hyperkube_for_arm

Automatic merge from submit-queue

Cross-build hyperkube and debian-iptables for ARM. Also add a flannel image

We have to be able to build complex docker images too on `amd64` hosts.
Right now we can't build Dockerfiles with `RUN` commands when building for other architectures e.g. ARM.

Resin has a tutorial about this here: https://resin.io/blog/building-arm-containers-on-any-x86-machine-even-dockerhub/
But it's a bit clumsy syntax.

The other alternative would be running this command in a Makefile:
```
# This registers in the kernel that ARM binaries should be run by /usr/bin/qemu-{ARCH}-static
docker run --rm --privileged multiarch/qemu-user-static:register --reset
```
and 
```
ADD https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-arm-static.tar.xz /usr/bin
```
Then the kernel will be able to differ ARM binaries from amd64. When it finds a ARM binary, it will invoke `/usr/bin/qemu-arm-static` first and lets `qemu` translate the ARM syscalls to amd64 ones.
Some code here: https://github.com/multiarch

WDYT is the best approach? If registering `binfmt_misc` in the kernels of the machines is OK, then I think we should go with that.
Otherwise, we'll have to wait for resin's patch to be merged into mainline qemu before we may use the code I have here now.

@fgrzadkowski @david-mcmahon @brendandburns @zmerlynn @ixdy @ihmccreery @thockin

build/common.sh

@@ -103,28 +103,28 @@ kube::build::get_docker_wrapped_binaries() {
           kube-apiserver,busybox
           kube-controller-manager,busybox
           kube-scheduler,busybox
-          kube-proxy,gcr.io/google_containers/debian-iptables:v1
+          kube-proxy,gcr.io/google_containers/debian-iptables-amd64:v2
         );;
-    "arm") # TODO: Use image with iptables installed for kube-proxy for arm, arm64 and ppc64le
+    "arm")
         local targets=(
-          kube-apiserver,hypriot/armhf-busybox
-          kube-controller-manager,hypriot/armhf-busybox
-          kube-scheduler,hypriot/armhf-busybox
-          kube-proxy,hypriot/armhf-busybox
+          kube-apiserver,armel/busybox
+          kube-controller-manager,armel/busybox
+          kube-scheduler,armel/busybox
+          kube-proxy,gcr.io/google_containers/debian-iptables-arm:v2
         );;
     "arm64")
         local targets=(
           kube-apiserver,aarch64/busybox
           kube-controller-manager,aarch64/busybox
           kube-scheduler,aarch64/busybox
-          kube-proxy,aarch64/busybox
+          kube-proxy,gcr.io/google_containers/debian-iptables-arm64:v2
         );;
     "ppc64le")
         local targets=(
           kube-apiserver,ppc64le/busybox
           kube-controller-manager,ppc64le/busybox
           kube-scheduler,ppc64le/busybox
-          kube-proxy,ppc64le/busybox
+          kube-proxy,gcr.io/google_containers/debian-iptables-ppc64le:v2
         );;
   esac
 
@@ -672,7 +672,12 @@ function kube::release::clean_cruft() {
 function kube::release::package_hyperkube() {
   # If we have these variables set then we want to build all docker images.
   if [[ -n "${KUBE_DOCKER_IMAGE_TAG-}" && -n "${KUBE_DOCKER_REGISTRY-}" ]]; then
-    REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" make -C cluster/images/hyperkube/ build
+    for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
+
+      local arch=${platform##*/}
+      kube::log::status "Building hyperkube image for arch: ${arch}"
+      REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" ARCH="${arch}" make -C cluster/images/hyperkube/ build
+    done
   fi
 }
 
@@ -738,7 +743,7 @@ function kube::release::package_client_tarballs() {
 # Package up all of the server binaries
 function kube::release::package_server_tarballs() {
   local platform
-  for platform in "${KUBE_SERVER_PLATFORMS[@]}" ; do
+  for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
     local platform_tag=${platform/\//-} # Replace a "/" for a "-"
     local arch=$(basename ${platform})
     kube::log::status "Building tarball: server $platform_tag"
@@ -1538,24 +1543,18 @@ function kube::release::docker::release() {
   for arch in "${archs[@]}"; do
     for binary in "${binaries[@]}"; do
 
-      # Temporary fix. hyperkube-arm isn't built in the release process, so we can't push it
-      # This if statement skips the push for hyperkube-arm
-      if [[ ${arch} != "arm" || ${binary} != "hyperkube" ]]; then
-
+      local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}"
+      kube::log::status "Pushing ${binary} to ${docker_target}"
+      "${docker_push_cmd[@]}" push "${docker_target}"
 
-        local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}"
-        kube::log::status "Pushing ${binary} to ${docker_target}"
-        "${docker_push_cmd[@]}" push "${docker_target}"
+      # If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions
+      if [[ ${arch} == "amd64" ]]; then
+        local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}"
 
-        # If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions
-        if [[ ${arch} == "amd64" ]]; then
-          local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}"
-
-          "${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null
+        "${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null
 
-          kube::log::status "Pushing ${binary} to ${legacy_docker_target}"
-          "${docker_push_cmd[@]}" push "${legacy_docker_target}"
-        fi
+        kube::log::status "Pushing ${binary} to ${legacy_docker_target}"
+        "${docker_push_cmd[@]}" push "${legacy_docker_target}"
       fi
     done
   done

build/debian-iptables/Dockerfile

@@ -12,10 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM debian:jessie
+FROM BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
 
 # All apt-get's must be in one run command or the
 # cleanup has no effect.
-RUN apt-get update && \
-    apt-get install -y iptables && \
-    ls /var/lib/apt/lists/*debian* | xargs rm
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y iptables \
+    && rm -rf /var/lib/apt/lists/*

build/debian-iptables/Makefile

@@ -14,13 +14,47 @@
 
 .PHONY:	build push
 
-IMAGE = debian-iptables
-TAG = v1
+REGISTRY?="gcr.io/google_containers"
+IMAGE=debian-iptables
+TAG=v2
+ARCH?=amd64
+TEMP_DIR:=$(shell mktemp -d)
+
+ifeq ($(ARCH),amd64)
+	BASEIMAGE?=debian:jessie
+endif
+ifeq ($(ARCH),arm)
+	BASEIMAGE?=armel/debian:jessie
+	QEMUARCH=arm
+endif
+ifeq ($(ARCH),arm64)
+	BASEIMAGE?=aarch64/debian:jessie
+	QEMUARCH=aarch64
+endif
+ifeq ($(ARCH),ppc64le)
+	BASEIMAGE?=ppc64le/debian:jessie
+	QEMUARCH=ppc64le
+endif
 
 build:
-	docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .
+	cp ./* $(TEMP_DIR)
+	cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
+	cd $(TEMP_DIR) && sed -i "s|ARCH|$(QEMUARCH)|g" Dockerfile
+
+ifeq ($(ARCH),amd64)
+	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" Dockerfile
+else
+	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-$(QEMUARCH)-static.tar.xz | tar -xJ -C $(TEMP_DIR)
+	cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" Dockerfile
+endif
+
+	docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
 
-push:	build
-	gcloud docker --server=gcr.io push gcr.io/google_containers/$(IMAGE):$(TAG)
+push: build
+	gcloud docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
 
-all:	push
+all: push

build/debian-iptables/README.md

@@ -0,0 +1,29 @@
+### debian-iptables
+
+Serves as the base image for `gcr.io/google_containers/kube-proxy-${ARCH}` and multiarch (not `amd64`) `gcr.io/google_containers/flannel-${ARCH}` images.
+
+This image is compiled for multiple architectures.
+
+#### How to release
+
+If you're editing the Dockerfile or some other thing, please bump the `TAG` in the Makefile.
+
+```console
+# Build for linux/amd64 (default)
+$ make push ARCH=amd64
+# ---> gcr.io/google_containers/debian-iptables-amd64:TAG
+
+$ make push ARCH=arm
+# ---> gcr.io/google_containers/debian-iptables-arm:TAG
+
+$ make push ARCH=arm64
+# ---> gcr.io/google_containers/debian-iptables-arm64:TAG
+
+$ make push ARCH=ppc64le
+# ---> gcr.io/google_containers/debian-iptables-ppc64le:TAG
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/build/debian-iptables/README.md?pixel)]()

cluster/images/flannel/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM BASEIMAGE
+
+COPY flanneld /opt/bin/
+COPY mk-docker-opts.sh /opt/bin/
+
+CMD ["/opt/bin/flanneld"]

cluster/images/flannel/Makefile

@@ -0,0 +1,60 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the flannel image
+#
+# Usage:
+# 	[TAG=0.5.5] [REGISTRY=gcr.io/google_containers] [ARCH=amd64] make build
+
+TAG?=0.5.5
+ARCH?=amd64
+REGISTRY?=gcr.io/google_containers
+KUBE_CROSS_TAG=v1.4.2-1
+GOARM=6
+TEMP_DIR:=$(shell mktemp -d)
+BASEIMAGE?=gcr.io/google_containers/debian-iptables-${ARCH}:v2
+
+ifeq ($(ARCH),arm)
+	CC=arm-linux-gnueabi-gcc
+endif
+
+build:
+ifeq ($(ARCH),amd64)
+	# If we should build an amd64 flannel, go with the official one
+	docker pull quay.io/coreos/flannel:$(TAG)
+
+	docker tag -f quay.io/coreos/flannel:$(TAG) $(REGISTRY)/flannel-$(ARCH):$(TAG)
+else
+	# Copy the content in this dir to the temp dir
+	cp ./* $(TEMP_DIR)
+
+	docker run -it -v $(TEMP_DIR):/flannel/bin gcr.io/google_containers/kube-cross:$(KUBE_CROSS_TAG) /bin/bash -c \
+    	"curl -sSL https://github.com/coreos/flannel/archive/v${TAG}.tar.gz | tar -C /flannel -xz --strip-components=1 \
+    	&& cd /flannel && GOARM=$(GOARM) GOARCH=$(ARCH) CC=$(CC) CGO_ENABLED=1 ./build"
+
+	# Replace BASEIMAGE with the real base image
+	cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
+
+	# Download mk-docker-opts.sh
+	curl -sSL https://raw.githubusercontent.com/coreos/flannel/v$(TAG)/dist/mk-docker-opts.sh > $(TEMP_DIR)/mk-docker-opts.sh
+
+	# And build the image
+	docker build -t $(REGISTRY)/flannel-$(ARCH):$(TAG) $(TEMP_DIR)
+endif
+
+push: build
+	gcloud docker push $(REGISTRY)/flannel-$(ARCH):$(TAG)
+
+all: build
+.PHONY: build push

cluster/images/flannel/README.md

@@ -0,0 +1,22 @@
+### flannel
+
+This is used mostly for the `docker-multinode` config, but also in other places where flannel runs in a container.
+
+For `amd64`, this image equals to `quay.io/coreos/flannel` to maintain official support.
+For other architectures, `flannel` is cross-compiled. The `debian-iptables` image serves as base image.
+
+#### How to release
+
+```console
+# Build for linux/amd64 (default)
+$ make push ARCH=amd64
+# ---> gcr.io/google_containers/flannel-amd64:TAG
+
+$ make push ARCH=arm
+# ---> gcr.io/google_containers/flannel-arm:TAG
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/flannel/README.md?pixel)]()

cluster/images/hyperkube/Dockerfile

@@ -14,6 +14,10 @@
 
 FROM BASEIMAGE
 
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
+
 RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
     && DEBIAN_FRONTEND=noninteractive apt-get -yy -q \
     install \

cluster/images/hyperkube/Makefile

@@ -15,17 +15,19 @@
 # Build the hyperkube image.
 #
 # Usage:
-#   VERSION=v1.1.2 [REGISTRY="gcr.io/google_containers"] make build
+#   VERSION=v1.2.0 [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make build
 
 REGISTRY?="gcr.io/google_containers"
-ARCH=amd64
-BASEIMAGE=debian:jessie
+ARCH?=amd64
 TEMP_DIR:=$(shell mktemp -d)
 
-## Comment in for arm builds, must be run on an arm machine
-# ARCH=arm
-# need to escape '/' for the regexp below
-# BASEIMAGE=armbuild\\/debian:jessie
+
+ifeq ($(ARCH),amd64)
+	BASEIMAGE?=debian:jessie
+endif
+ifeq ($(ARCH),arm)
+	BASEIMAGE?=armel/debian:jessie
+endif
 
 all: build
 
@@ -38,18 +40,27 @@ endif
 	cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
 	cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
 	cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json
-	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json
+	cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json Dockerfile
 	cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
 	rm ${TEMP_DIR}/*.back
-	docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR}
-	# Backward compatibility.  TODO: deprecate this image tag
+
 ifeq ($(ARCH),amd64)
-	docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION}
+	# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	cd ${TEMP_DIR} && sed -i "/CROSS_BUILD_/d" Dockerfile
+else
+	# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+	# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${ARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR}
+	cd ${TEMP_DIR} && sed -i "s/CROSS_BUILD_//g" Dockerfile
 endif
 
+	docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR}
+
 push: build
 	gcloud docker push ${REGISTRY}/hyperkube-${ARCH}:${VERSION}
 ifeq ($(ARCH),amd64)
+	docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION}
 	gcloud docker push ${REGISTRY}/hyperkube:${VERSION}
 endif
 

cluster/images/hyperkube/README.md

@@ -0,0 +1,27 @@
+### hyperkube
+
+`hyperkube` is an all-in-one binary for the Kubernetes server components
+Also, it's very easy to run this `hyperkube` setup dockerized.
+See http://kubernetes.io/docs/getting-started-guides/docker/ for up-to-date commands.
+
+`hyperkube` is built for multiple architectures and pushed on every release.
+
+#### How to release by hand
+
+```console
+# First, build the 
+$ build/run.sh hack/build-cross.sh
+
+# Build for linux/amd64 (default)
+$ make push VERSION={target_version} ARCH=amd64
+# ---> gcr.io/google_containers/hyperkube-amd64:VERSION
+# ---> gcr.io/google_containers/hyperkube:VERSION (image with backwards-compatible naming)
+
+$ make push VERSION={target_version} ARCH=arm
+# ---> gcr.io/google_containers/hyperkube-arm:VERSION
+```
+
+If you don't want to push the images, run `make` or `make build` instead
+
+
+[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/hyperkube/README.md?pixel)]()