Merge pull request kubernetes-sigs#579 from jkh52/cross-compile

Change Dockerfiles (and make targets) to use cross-compile.
jkh52 · Mar 12, 2024 · 26b0440 · 26b0440
2 parents 8ff71dc + 74ea854
commit 26b0440
Show file tree

Hide file tree

Showing 8 changed files with 50 additions and 37 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -36,10 +36,10 @@ jobs:
  run: |
  mkdir _output
 
- docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} .
- docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
- docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} .
- docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar
+ make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
+ docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local > _output/konnectivity-agent.tar
+ make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
+ docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local > _output/konnectivity-server.tar
 
  - uses: actions/upload-artifact@v2
  with:
@@ -150,8 +150,8 @@ jobs:
  # preload konnectivity images
  docker load --input konnectivity-server.tar
  docker load --input konnectivity-agent.tar
- /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
- /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
+ /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local --name ${{ env.KIND_CLUSTER_NAME}}
+ /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local --name ${{ env.KIND_CLUSTER_NAME}}
  kubectl apply -f examples/kind/konnectivity-server.yaml
  kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
 

diff --git a/Makefile b/Makefile
@@ -12,7 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# ARCH specifies the golang target platform, for docker build
+# targetes. It is supported for backward compatibility, but prefer
+# BUILDARCH and TARGETARCH.
 ARCH ?= amd64
+# BUILDARCH specifies the golang build platform, for docker build targets.
+BUILDARCH ?= amd64
+# TARGETARCH specifies the golang target platform, for docker build targets.
+TARGETARCH ?= $(ARCH)
 ARCH_LIST ?= amd64 arm arm64 ppc64le s390x
 RELEASE_ARCH_LIST = amd64 arm64
 # The output type could either be docker (local), or registry.
@@ -201,53 +208,53 @@ docker-push: docker-push/proxy-agent docker-push/proxy-server
 .PHONY: docker-build/proxy-agent
 docker-build/proxy-agent: cmd/agent/main.go proto/agent/agent.pb.go buildx-setup
  @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
- echo "Building proxy-agent for ${ARCH}"
- ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
+ echo "Building proxy-agent with ${BUILDARCH} for ${TARGETARCH}"
+ ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-agent
 docker-push/proxy-agent: docker-build/proxy-agent
  @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
- ${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
+ ${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/proxy-server
 docker-build/proxy-server: cmd/server/main.go proto/agent/agent.pb.go buildx-setup
  @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
- echo "Building proxy-server for ${ARCH}"
- ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+ echo "Building proxy-server with ${BUILDARCH} for ${TARGETARCH}"
+ ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-server
 docker-push/proxy-server: docker-build/proxy-server
  @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
- ${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+ ${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/proxy-test-client
 docker-build/proxy-test-client: cmd/test-client/main.go proto/agent/agent.pb.go buildx-setup
  @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
- echo "Building proxy-test-client for ${ARCH}"
- ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+ echo "Building proxy-test-client with ${BUILDARCH} for ${TARGETARCH}"
+ ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-test-client
 docker-push/proxy-test-client: docker-build/proxy-test-client
  @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
- ${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+ ${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/http-test-server
 docker-build/http-test-server: cmd/test-server/main.go buildx-setup
  @[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
- echo "Building http-test-server for ${ARCH}"
- ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+ echo "Building http-test-server with ${BUILDARCH} for ${TARGETARCH}"
+ ${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/http-test-server
 docker-push/http-test-server: docker-build/http-test-server
  @[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
- ${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+ ${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 ## --------------------------------------
 ## Docker — All ARCH
 ## --------------------------------------
 
 # As `docker buildx` is time and resource consuming, if not necessary, building specific arch images,
-# like `make docker-build-arch-aamd64`, is recommended.
+# like `make docker-build-arch-amd64`, is recommended.
 .PHONY: docker-build-all
 docker-build-all: $(addprefix docker-build-arch-,$(ARCH_LIST))
 
@@ -261,16 +268,16 @@ docker-build-arch-%:
  $(MAKE) docker-build/proxy-server-$*
 
 docker-build/proxy-agent-%:
- $(MAKE) ARCH=$* docker-build/proxy-agent
+ $(MAKE) TARGETARCH=$* docker-build/proxy-agent
 
 docker-push/proxy-agent-%:
- $(MAKE) ARCH=$* docker-push/proxy-agent
+ $(MAKE) TARGETARCH=$* docker-push/proxy-agent
 
 docker-build/proxy-server-%:
- $(MAKE) ARCH=$* docker-build/proxy-server
+ $(MAKE) TARGETARCH=$* docker-build/proxy-server
 
 docker-push/proxy-server-%:
- $(MAKE) ARCH=$* docker-push/proxy-server
+ $(MAKE) TARGETARCH=$* docker-push/proxy-server
 
 .PHONY: docker-push-manifest/proxy-agent
 docker-push-manifest/proxy-agent: ## Push the fat manifest docker image.

diff --git a/artifacts/images/agent-build.Dockerfile b/artifacts/images/agent-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the proxy-agent binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -26,8 +27,8 @@ COPY proto/ proto/
 
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/server-build.Dockerfile b/artifacts/images/server-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the proxy-server binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -25,8 +26,8 @@ COPY cmd/ cmd/
 COPY proto/ proto/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/test-client-build.Dockerfile b/artifacts/images/test-client-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the client binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -25,8 +26,8 @@ COPY cmd/ cmd/
 COPY proto/ proto/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/test-server-build.Dockerfile b/artifacts/images/test-server-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the http test server binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -24,8 +25,8 @@ COPY pkg/ pkg/
 COPY cmd/ cmd/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server
 
 FROM ${BASEIMAGE}
 

diff --git a/examples/kind/konnectivity-agent-ds.yaml b/examples/kind/konnectivity-agent-ds.yaml
@@ -36,7 +36,8 @@ spec:
  dnsPolicy: ClusterFirstWithHostNet
  containers:
  - name: konnectivity-agent-container
- image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
+ image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local
+ imagePullPolicy: Never
  resources:
  requests:
  cpu: 50m

diff --git a/examples/kind/konnectivity-server.yaml b/examples/kind/konnectivity-server.yaml
@@ -55,7 +55,8 @@ spec:
  hostNetwork: true
  containers:
  - name: konnectivity-server-container
- image: gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
+ image: gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local
+ imagePullPolicy: Never
  resources:
  requests:
  cpu: 1m