embed runtime files

Makefile

@@ -3,39 +3,71 @@ PKG := "github.com/OpenFogStack/$(PROJECT_NAME)"
 GO_FILES := $(shell find . -name '*.go' | grep -v /vendor/ | grep -v /ext/ | grep -v _test.go)
 TEST_DIR := ./test
 
+SUPPORTED_ARCH=amd64 arm arm64
+RUNTIMES := $(shell find pkg/docker/runtimes -name Dockerfile | xargs -n1 dirname | xargs -n1 basename)
+
 OS=$(shell go env GOOS)
 ARCH=$(shell go env GOARCH)
 
-.PHONY: all build start clean
-
+.PHONY: all
 all: build
 
+.PHONY: build
 build: tinyfaas-${OS}-${ARCH}
 
+.PHONY: start
+start: tinyfaas-${OS}-${ARCH}
+	./$<
+
+.PHONY: test
+test: build ${TEST_DIR}/test_all.py
+	@python3 ${TEST_DIR}/test_all.py
+
+.PHONY: clean
+clean: clean.sh
+	@sh clean.sh
+
+
+define arch_build
+pkg/docker/runtimes-$(arch): $(foreach runtime,$(RUNTIMES),pkg/docker/runtimes-$(arch)/$(runtime))
+endef
+$(foreach arch,$(SUPPORTED_ARCH),$(eval $(arch_build)))
+
+define runtime_build
+.PHONY: pkg/docker/runtimes-$(arch)/$(runtime)
+pkg/docker/runtimes-$(arch)/$(runtime): pkg/docker/runtimes-$(arch)/$(runtime)/Dockerfile pkg/docker/runtimes-$(arch)/$(runtime)/blob.tar.gz
+
+pkg/docker/runtimes-$(arch)/$(runtime)/blob.tar.gz: pkg/docker/runtimes/$(runtime)/build.Dockerfile
+	mkdir -p $$(@D)
+	cd $$(<D) ; docker build --platform=linux/$(arch) -t tf-build-$(arch)-$(runtime) -f $$(<F) .
+	docker run -d -t --platform=linux/$(arch) --name $${PROJECT_NAME}-$(runtime) --rm tf-build-$(arch)-$(runtime)
+	docker export $${PROJECT_NAME}-$(runtime) | gzip > $$@
+	docker kill $${PROJECT_NAME}-$(runtime)
+
+pkg/docker/runtimes-$(arch)/$(runtime)/Dockerfile: pkg/docker/runtimes/$(runtime)/Dockerfile
+	mkdir -p $$(@D)
+	cp -r pkg/docker/runtimes/$(runtime)/Dockerfile $$@
+endef
+$(foreach arch,$(SUPPORTED_ARCH),$(foreach runtime,$(RUNTIMES),$(eval $(runtime_build))))
+
 # requires protoc,  protoc-gen-go and protoc-gen-go-grpc
 # install from your package manager, e.g.:
 # 	brew install protobuf
 # 	brew install protoc-gen-go
 #	brew install protoc-gen-go-grpc
 pkg/grpc/tinyfaas/tinyfaas.pb.go pkg/grpc/tinyfaas/tinyfaas_grpc.pb.go: pkg/grpc/tinyfaas/tinyfaas.proto
-	@protoc -I . $< --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,paths=source_relative
+	@protoc -I $(<D) $< --go_out=$(<D) --go_opt=paths=source_relative --go-grpc_out=$(<D) --go-grpc_opt=require_unimplemented_servers=false,paths=source_relative
 
 # requires grpcio-tools and mypy-protobuf
 # 	python3 -m pip install -r requirements.txt
 pkg/grpc/tinyfaas/tinyfaas_pb2.py pkg/grpc/tinyfaas/tinyfaas_pb2.pyi pkg/grpc/tinyfaas/tinyfaas_pb2_grpc.py: pkg/grpc/tinyfaas/tinyfaas.proto
-	@python3 -m grpc_tools.protoc -I . --python_out=. --grpc_python_out=. --mypy_out=. $<
+	@python3 -m grpc_tools.protoc -I $(<D) --python_out=$(<D) --grpc_python_out=$(<D) --mypy_out=$(<D) $<
 
 cmd/manager/rproxy-%.bin: pkg/grpc/tinyfaas/tinyfaas_pb2.py pkg/grpc/tinyfaas/tinyfaas_pb2.pyi pkg/grpc/tinyfaas/tinyfaas_pb2_grpc.py pkg/grpc/tinyfaas/tinyfaas.pb.go pkg/grpc/tinyfaas/tinyfaas_grpc.pb.go $(GO_FILES)
 	GOOS=$(word 1,$(subst -, ,$*)) GOARCH=$(word 2,$(subst -, ,$*)) go build -o $@ -v $(PKG)/cmd/rproxy
 
-tinyfaas-%: cmd/manager/rproxy-%.bin $(GO_FILES)
-	GOOS=$(word 1,$(subst -, ,$*)) GOARCH=$(word 2,$(subst -, ,$*)) go build -o $@ -v $(PKG)/cmd/manager
-
-start: tinyfaas-${OS}-${ARCH}
-	./$<
-
-test: build ${TEST_DIR}/test_all.py
-	@python3 ${TEST_DIR}/test_all.py
+tinyfaas-darwin-%: cmd/manager/rproxy-darwin-%.bin pkg/docker/runtimes-% $(GO_FILES)
+	GOOS=darwin GOARCH=$* go build -o $@ -v $(PKG)/cmd/manager
 
-clean: clean.sh
-	@sh clean.sh
+tinyfaas-linux-%: cmd/manager/rproxy-linux-%.bin pkg/docker/runtimes-% $(GO_FILES)
+	GOOS=linux GOARCH=$* go build -o $@ -v $(PKG)/cmd/manager

cmd/manager/rproxy-linux-arm.go

@@ -1,5 +1,5 @@
-//go:build linux && armv7
-// +build linux,armv7
+//go:build linux && arm
+// +build linux,arm
 
 package main
 

pkg/docker/handler.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"embed"
 	"fmt"
 	"io"
 	"log"
@@ -24,9 +23,6 @@ import (
 	"github.com/google/uuid"
 )
 
-//go:embed runtimes
-var runtimes embed.FS
-
 const (
 	TmpDir           = "./tmp"
 	containerTimeout = 1
@@ -99,7 +95,7 @@ func (db *DockerBackend) Create(name string, env string, threads int, filedir st
 	// copy Docker stuff into folder
 	// cp runtimes/<env>/* <folder>
 
-	err = util.CopyDirFromEmbed(runtimes, path.Join("runtimes", dh.env), dh.filePath)
+	err = util.CopyDirFromEmbed(runtimes, path.Join(runtimesDir, dh.env), dh.filePath)
 	if err != nil {
 		return nil, err
 	}

pkg/docker/runtimes-amd64.go

@@ -0,0 +1,11 @@
+//go:build amd64
+// +build amd64
+
+package docker
+
+import "embed"
+
+//go:embed runtimes-amd64
+var runtimes embed.FS
+
+const runtimesDir = "runtimes-amd64"

pkg/docker/runtimes-amd64/binary/Dockerfile

@@ -0,0 +1,13 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY . .
+RUN mv fn/* .
+RUN chmod +x fn.sh
+
+CMD [ "./handler.bin" ]

pkg/docker/runtimes-amd64/nodejs/Dockerfile

@@ -0,0 +1,13 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+RUN npm install ./fn
+
+CMD [ "node", "functionhandler.js" ]

pkg/docker/runtimes-amd64/python3/Dockerfile

@@ -0,0 +1,14 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY fn/* ./
+RUN python -m pip install -r requirements.txt --user
+
+ENV PYTHONUNBUFFERED=1
+CMD [ "python3", "functionhandler.py" ]

pkg/docker/runtimes-arm.go

@@ -0,0 +1,11 @@
+//go:build arm
+// +build arm
+
+package docker
+
+import "embed"
+
+//go:embed runtimes-arm
+var runtimes embed.FS
+
+const runtimesDir = "runtimes-arm"

pkg/docker/runtimes-arm/binary/Dockerfile

@@ -0,0 +1,12 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY fn/* ./
+RUN chmod +x fn.sh
+
+CMD [ "./handler.bin" ]

pkg/docker/runtimes-arm/nodejs/Dockerfile

@@ -0,0 +1,13 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY . .
+
+RUN npm install ./fn
+
+CMD [ "node", "functionhandler.js" ]

pkg/docker/runtimes-arm/python3/Dockerfile

@@ -0,0 +1,14 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY fn/* ./
+RUN python -m pip install -r requirements.txt --user
+
+ENV PYTHONUNBUFFERED=1
+CMD [ "python3", "functionhandler.py" ]

pkg/docker/runtimes-arm64.go

@@ -0,0 +1,11 @@
+//go:build arm64
+// +build arm64
+
+package docker
+
+import "embed"
+
+//go:embed runtimes-arm64
+var runtimes embed.FS
+
+const runtimesDir = "runtimes-arm64"

pkg/docker/runtimes-arm64/binary/Dockerfile

@@ -0,0 +1,12 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY fn/* ./
+RUN chmod +x fn.sh
+
+CMD [ "./handler.bin" ]

pkg/docker/runtimes-arm64/nodejs/Dockerfile

@@ -0,0 +1,13 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+WORKDIR /usr/src/app
+
+COPY fn fn
+
+RUN npm install ./fn
+
+CMD [ "node", "functionhandler.js" ]

pkg/docker/runtimes-arm64/python3/Dockerfile

@@ -0,0 +1,14 @@
+FROM scratch
+
+ADD blob.tar.gz /
+
+EXPOSE 8000
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY fn/* ./
+RUN python -m pip install -r requirements.txt --user
+
+ENV PYTHONUNBUFFERED=1
+CMD [ "python3", "functionhandler.py" ]

pkg/docker/runtimes/binary/Dockerfile

@@ -1,21 +1,12 @@
-FROM golang:1.20-alpine AS builder
+FROM scratch
 
-WORKDIR /usr/src/build
-COPY functionhandler.go .
-RUN GO111MODULE=off CGO_ENABLED=0 go build -o handler.bin .
-
-FROM alpine
+ADD blob.tar.gz /
 
 EXPOSE 8000
 
-# Create app directory
 WORKDIR /usr/src/app
 
-COPY --from=builder /usr/src/build/handler.bin .
-
-COPY . .
-RUN rm functionhandler.go
-RUN mv fn/* .
+COPY fn/* ./
 RUN chmod +x fn.sh
 
 CMD [ "./handler.bin" ]

pkg/docker/runtimes/binary/build.Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.22-alpine AS builder
+
+WORKDIR /usr/src/build
+COPY functionhandler.go .
+RUN GO111MODULE=off CGO_ENABLED=0 go build -o handler.bin .
+
+FROM alpine:3.19
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY --from=builder /usr/src/build/handler.bin .

pkg/docker/runtimes/nodejs/Dockerfile

@@ -1,15 +1,13 @@
-#https://nodejs.org/en/docs/guides/nodejs-docker-webapp/
-FROM node:20-alpine
+FROM scratch
+
+ADD blob.tar.gz /
 
 EXPOSE 8000
 
-# Create app directory
 WORKDIR /usr/src/app
-RUN npm cache add express
 
-COPY . .
-RUN npm install express
-RUN npm install body-parser
+COPY fn fn
+
 RUN npm install ./fn
 
 CMD [ "node", "functionhandler.js" ]

pkg/docker/runtimes/nodejs/build.Dockerfile

@@ -0,0 +1,12 @@
+#https://nodejs.org/en/docs/guides/nodejs-docker-webapp/
+FROM node:20-alpine
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY functionhandler.js .
+COPY package.json .
+
+RUN npm install express && \
+    npm install body-parser && \
+    npm cache clean --force

pkg/docker/runtimes/python3/Dockerfile

@@ -1,13 +1,13 @@
-FROM python:3.11-alpine
+FROM scratch
 
+ADD blob.tar.gz /
 
 EXPOSE 8000
 
 # Create app directory
 WORKDIR /usr/src/app
 
-COPY . .
-RUN mv fn/* .
+COPY fn/* ./
 RUN python -m pip install -r requirements.txt --user
 
 ENV PYTHONUNBUFFERED=1

pkg/docker/runtimes/python3/build.Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.11-alpine
+
+# Create app directory
+WORKDIR /usr/src/app
+
+COPY functionhandler.py .

pkg/docker/runtimes/python3/functionhandler.py

@@ -6,7 +6,7 @@
 
 if __name__ == "__main__":
     try:
-        import fn
+        import fn  # type: ignore
     except ImportError:
         raise ImportError("Failed to import fn.py")
 
@@ -36,7 +36,9 @@ def do_POST(self) -> None:
                 res = fn.fn(d)
                 self.send_response(200)
                 self.end_headers()
-                self.wfile.write(res.encode("utf-8"))
+                if res is not None:
+                    self.wfile.write(res.encode("utf-8"))
+
                 return
             except Exception as e:
                 print(e)