From 87d003fd1fe3b84a9d92cdfadb222ad2fa71f1b3 Mon Sep 17 00:00:00 2001 From: Bernd Ahlers Date: Wed, 25 Oct 2023 17:19:58 +0200 Subject: [PATCH] Add ARM64 binary and package builds (#480) - Add ARM64 binary and package builds - Download ARM64 beats and include them in the OS package - Add filebeat and auditbeat to the dist tarball - Cleanup Makefile - Put .PHONY directly in front of the targets - Add architecture names to each target - Remove no-op solaris patch download. We switched to elastic/gosigar and the patch is installed for cloudfoundry/gosigar. - Make build less verbose by removing the -v flag from go build - Make *-all targets more readable - Fix solaris build - Let fetch_collectors.sh fail early Co-authored-by: Marco Pfatschbacher --- Makefile | 109 ++++++++++++++++++++++--------- api/version.go | 15 +++++ changelog/unreleased/pr-480.toml | 4 ++ common/sigar.go | 2 +- common/sigar_solaris.go | 28 ++++++++ daemon/svc_helper.go | 2 +- dist/fetch_collectors.sh | 8 ++- dist/recipearm64.rb | 34 ++++++++++ services/control_handler_unix.go | 2 +- 9 files changed, 169 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/pr-480.toml create mode 100644 common/sigar_solaris.go create mode 100644 dist/recipearm64.rb diff --git a/Makefile b/Makefile index 0eab6ec0..be8cdb12 100644 --- a/Makefile +++ b/Makefile @@ -22,114 +22,154 @@ WINDOWS_INSTALLER_VERSION = $(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)$(subst -, # Removing the dot to comply with NuGet versioning (beta.1 -> beta2) CHOCOLATEY_VERSION = $(COLLECTOR_VERSION).$(COLLECTOR_REVISION)$(subst .,,$(COLLECTOR_VERSION_SUFFIX)) +.PHONY: all all: build +.PHONY: fmt fmt: ## Run gofmt @GOFMT=$(GOFMT) sh ./format.sh +.PHONY: clean clean: ## Remove binaries -rm -rf $(targets) +.PHONY: distclean distclean: clean -rm -rf $(dist_targets) +.PHONY: test test: ## Run tests $(GO) test -v $(TEST_SUITE) +.PHONY: build build: ## Build sidecar binary for local target system - $(GO) build $(BUILD_OPTS) -v -o graylog-sidecar + $(GO) build $(BUILD_OPTS) -o graylog-sidecar -build-all: build-linux-armv7 build-linux build-linux32 build-windows build-windows32 build-darwin build-darwin-arm64 build-freebsd +.PHONY: build-all +build-all: build-linux-amd64 build-linux-arm64 build-linux-armv7 build-linux32 +build-all: build-darwin-amd64 build-darwin-arm64 +build-all: build-freebsd-amd64 +build-all: build-windows-amd64 build-windows32 -build-linux: ## Build sidecar binary for Linux +.PHONY: build-linux-amd64 +build-linux-amd64: ## Build sidecar binary for linux-amd64 @mkdir -p build/$(COLLECTOR_VERSION)/linux/amd64 - GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar + GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar -solaris-sigar-patch: - # https://github.com/cloudfoundry/gosigar/pull/28 - @if [ ! -e vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go ]; then \ - wget -O vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go https://raw.githubusercontent.com/amitkris/gosigar/9fc0903125acd1a0dc7635f8670088339865bcd5/sigar_solaris.go; \ - fi +.PHONY: build-linux-arm64 +build-linux-arm64: ## Build sidecar binary for linux-arm64 + @mkdir -p build/$(COLLECTOR_VERSION)/linux/arm64 + GOOS=linux GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-arm64 -o build/$(COLLECTOR_VERSION)/linux/arm64/graylog-sidecar +.PHONY: build-linux-armv7 build-linux-armv7: ## Build sidecar binary for linux-armv7 @mkdir -p build/$(COLLECTOR_VERSION)/linux/armv7 - GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7 -v -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar - -build-solaris: solaris-sigar-patch ## Build sidecar binary for Solaris/OmniOS/Illumos - @mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64 - GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar + GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7 -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar +.PHONY: build-linux32 build-linux32: ## Build sidecar binary for Linux 32bit @mkdir -p build/$(COLLECTOR_VERSION)/linux/386 - GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32 -v -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar + GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32 -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar -build-darwin: ## Build sidecar binary for OSX +.PHONY: build-darwin-amd64 +build-darwin-amd64: ## Build sidecar binary for OSX @mkdir -p build/$(COLLECTOR_VERSION)/darwin/amd64 - GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -v -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar + GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar +.PHONY: build-darwin-arm64 build-darwin-arm64: ## Build sidecar binary for OSX @mkdir -p build/$(COLLECTOR_VERSION)/darwin/arm64 - GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -v -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar + GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar -build-freebsd: ## Build sidecar binary for FreeBSD +.PHONY: build-freebsd-amd64 +build-freebsd-amd64: ## Build sidecar binary for FreeBSD @mkdir -p build/$(COLLECTOR_VERSION)/freebsd/amd64 - GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -v -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar + GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar -build-windows: install-goversioninfo ## Build sidecar binary for Windows +.PHONY: build-windows-amd64 +build-windows-amd64: install-goversioninfo ## Build sidecar binary for Windows @mkdir -p build/$(COLLECTOR_VERSION)/windows/amd64 $(GOVERSIONINFO_BIN) -64 -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso - GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -v -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe + GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe +.PHONY: build-windows32 build-windows32: install-goversioninfo ## Build sidecar binary for Windows 32bit @mkdir -p build/$(COLLECTOR_VERSION)/windows/386 $(GOVERSIONINFO_BIN) -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso - GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -v -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe + GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe + +.PHONY: build-solaris +build-solaris: ## Build sidecar binary for Solaris/OmniOS/Illumos + @mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64 + GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar +.PHONY: sign-binaries sign-binaries: sign-binary-windows-amd64 sign-binary-windows-386 +.PHONY: sign-binary-windows-amd64 sign-binary-windows-amd64: # This needs to run in a Docker container with the graylog/internal-codesigntool image codesigntool sign build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe +.PHONY: sign-binary-windows-386 sign-binary-windows-386: # This needs to run in a Docker container with the graylog/internal-codesigntool image codesigntool sign build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe ## Adds version info to Windows executable +.PHONY: install-goversioninfo install-goversioninfo: go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest -package-all: prepare-package package-linux-armv7 package-linux package-linux32 package-windows package-tar +.PHONY: package-all +package-all: prepare-package +package-all: package-linux-armv7 package-linux-arm64 package-linux-amd64 package-linux32 +package-all: package-windows-amd64 +package-all: package-tar +.PHONY: prepare-package prepare-package: dist/fetch_collectors.sh +.PHONY: package-linux-armv7 package-linux-armv7: ## Create Linux ARMv7 system package fpm-cook clean dist/recipearmv7.rb rm -rf dist/cache dist/tmp-build dist/tmp-dest fpm-cook -t deb package dist/recipearmv7.rb fpm-cook -t rpm package dist/recipearmv7.rb -package-linux: ## Create Linux amd64 system package +.PHONY: package-linux-arm64 +package-linux-arm64: ## Create Linux ARM64 system package + fpm-cook clean dist/recipearm64.rb + rm -rf dist/cache dist/tmp-build dist/tmp-dest + fpm-cook -t deb package dist/recipearm64.rb + fpm-cook -t rpm package dist/recipearm64.rb + +.PHONY: package-linux-amd64 +package-linux-amd64: ## Create Linux amd64 system package fpm-cook clean dist/recipe.rb rm -rf dist/cache dist/tmp-build dist/tmp-dest fpm-cook -t deb package dist/recipe.rb fpm-cook -t rpm package dist/recipe.rb +.PHONY: package-linux32 package-linux32: ## Create Linux i386 system package fpm-cook clean dist/recipe32.rb rm -rf dist/cache dist/tmp-build dist/tmp-dest fpm-cook -t deb package dist/recipe32.rb fpm-cook -t rpm package dist/recipe32.rb -package-windows: prepare-package ## Create Windows installer +.PHONY: package-windows-amd64 +package-windows-amd64: prepare-package ## Create Windows installer @mkdir -p dist/pkg makensis -DVERSION=$(COLLECTOR_VERSION) -DVERSION_SUFFIX=$(COLLECTOR_VERSION_SUFFIX) -DREVISION=$(COLLECTOR_REVISION) dist/recipe.nsi +.PHONY: sign-windows-installer sign-windows-installer: # This needs to run in a Docker container with the graylog/internal-codesigntool image codesigntool sign dist/pkg/graylog_sidecar_installer_$(WINDOWS_INSTALLER_VERSION).exe +.PHONY: package-chocolatey package-chocolatey: ## Create Chocolatey .nupkg file # This needs to run in a Docker container based on the Dockerfile.chocolatey image! dist/chocolatey/gensha.sh $(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX) $(WINDOWS_INSTALLER_VERSION) @@ -137,18 +177,27 @@ package-chocolatey: ## Create Chocolatey .nupkg file # See: https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#where-nugetversion-diverges-from-semantic-versioning cd dist/chocolatey && choco pack graylog-sidecar.nuspec --version $(CHOCOLATEY_VERSION) --out ../pkg +.PHONY: push-chocolatey push-chocolatey: ## Push Chocolatey .nupkg file # This needs to run in a Docker container based on the Dockerfile.chocolatey image! # Escape the CHOCO_API_KEY to avoid printing it in the logs! choco push dist/pkg/graylog-sidecar.$(CHOCOLATEY_VERSION).nupkg -k=$$CHOCO_API_KEY +.PHONY: package-tar package-tar: ## Create tar archive for all platforms @mkdir -p dist/pkg - @tar --transform="s|/build|/graylog-sidecar|" -Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz ./build ./sidecar-example.yml ./sidecar-windows-example.yml - + @tar --transform="s|/build|/graylog-sidecar|" --transform="s|/dist|/graylog-sidecar|" \ + -Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz \ + ./build \ + ./dist/collectors/auditbeat/linux/arm64/auditbeat \ + ./dist/collectors/auditbeat/linux/x86_64/auditbeat \ + ./dist/collectors/filebeat/linux/arm64/filebeat \ + ./dist/collectors/filebeat/linux/x86_64/filebeat \ + ./sidecar-example.yml \ + ./sidecar-windows-example.yml + +.PHONY: help help: @grep -hE '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | $(AWK) 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' .DEFAULT_GOAL := all - -.PHONY: all build build-all build-linux build-linux32 build-darwin build-freebsd build-windows build-windows32 fmt clean distclean help package-all package-linux package-linux32 package-windows package-tar diff --git a/api/version.go b/api/version.go index 6ea04cf1..8dc65da1 100644 --- a/api/version.go +++ b/api/version.go @@ -1,3 +1,18 @@ +// Copyright (C) 2020 Graylog, Inc. + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the Server Side Public License, version 1, +// as published by MongoDB, Inc. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// Server Side Public License for more details. +// +// You should have received a copy of the Server Side Public License +// along with this program. If not, see +// . + package api import "github.com/hashicorp/go-version" diff --git a/changelog/unreleased/pr-480.toml b/changelog/unreleased/pr-480.toml new file mode 100644 index 00000000..c0810bda --- /dev/null +++ b/changelog/unreleased/pr-480.toml @@ -0,0 +1,4 @@ +type = "a" +message = "Add operating system packages for Linux ARM64." + +pulls = ["480"] diff --git a/common/sigar.go b/common/sigar.go index ba44a58b..683fd4e1 100644 --- a/common/sigar.go +++ b/common/sigar.go @@ -13,7 +13,7 @@ // along with this program. If not, see // . -// +build !freebsd,!darwin +//go:build !freebsd && !darwin && !solaris package common diff --git a/common/sigar_solaris.go b/common/sigar_solaris.go new file mode 100644 index 00000000..4197db03 --- /dev/null +++ b/common/sigar_solaris.go @@ -0,0 +1,28 @@ +// Copyright (C) 2020 Graylog, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the Server Side Public License, version 1, +// as published by MongoDB, Inc. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// Server Side Public License for more details. +// +// You should have received a copy of the Server Side Public License +// along with this program. If not, see +// . + +package common + +func GetCpuIdle() float64 { + return -1 +} + +func GetFileSystemList75(string) []string { + return []string{} +} + +func GetLoad1() float64 { + return -1 +} diff --git a/daemon/svc_helper.go b/daemon/svc_helper.go index 154ca0ad..ce96fbf5 100644 --- a/daemon/svc_helper.go +++ b/daemon/svc_helper.go @@ -13,7 +13,7 @@ // along with this program. If not, see // . -// +build !windows +//go:build !windows package daemon diff --git a/dist/fetch_collectors.sh b/dist/fetch_collectors.sh index 5432e7c1..f6b0a97b 100755 --- a/dist/fetch_collectors.sh +++ b/dist/fetch_collectors.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -eo pipefail + FILEBEAT_VERSION=8.9.0 FILEBEAT_VERSION_32=7.17.12 WINLOGBEAT_VERSION=8.9.0 @@ -24,7 +26,7 @@ download_beat() archive="/tmp/${name}-${version}-${os}-${arch}.zip" if [ ! -f $archive ]; then echo "==> Downloading ${name}-${version}-${os}-${arch}" - curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip + curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip fi unzip -o -d dist/collectors/${name}/${os}/${arch} $archive mv dist/collectors/${name}/${os}/${arch}/${name}-${version}-${os}-${arch}/* dist/collectors/${name}/${os}/${arch}/ @@ -34,7 +36,7 @@ download_beat() archive="/tmp/${name}-${version}-${os}-${arch}.tar.gz" if [ ! -f $archive ]; then echo "==> Downloading ${name}-${version}-${os}-${arch}" - curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz + curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz fi tar -xzf $archive --strip-components=1 -C dist/collectors/${name}/${os}/${arch} ;; @@ -43,9 +45,11 @@ download_beat() download_beat "filebeat" "linux" ${FILEBEAT_VERSION} x86_64 download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} x86 +download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} arm64 download_beat "auditbeat" "linux" ${FILEBEAT_VERSION} x86_64 download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} x86 +download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} arm64 download_beat "filebeat" "windows" ${FILEBEAT_VERSION} x86_64 download_beat "filebeat" "windows" ${FILEBEAT_VERSION_32} x86 diff --git a/dist/recipearm64.rb b/dist/recipearm64.rb new file mode 100644 index 00000000..b96b4a4e --- /dev/null +++ b/dist/recipearm64.rb @@ -0,0 +1,34 @@ +require_relative 'tools' + +class GraylogSidecar < FPM::Cookery::Recipe + description 'Graylog collector sidecar' + + name 'graylog-sidecar' + version data.version + revision data.revision + homepage 'https://graylog.org' + arch 'arm64' + + source "file:../../build/#{version}/linux/arm64/graylog-sidecar" + + maintainer 'Graylog, Inc. ' + vendor 'graylog' + license 'SSPL' + + config_files '/etc/graylog/sidecar/sidecar.yml' + + fpm_attributes rpm_os: 'linux' + + def build + end + + def install + bin.install 'graylog-sidecar' + lib('graylog-sidecar').install '../../collectors/filebeat/linux/arm64/filebeat' + lib('graylog-sidecar').install '../../collectors/auditbeat/linux/arm64/auditbeat' + etc('graylog/sidecar').install '../../../sidecar-example.yml', 'sidecar.yml' + var('lib/graylog-sidecar/generated').mkdir + var('log/graylog-sidecar').mkdir + var('run/graylog-sidecar').mkdir + end +end diff --git a/services/control_handler_unix.go b/services/control_handler_unix.go index a8477fe2..fe92e965 100644 --- a/services/control_handler_unix.go +++ b/services/control_handler_unix.go @@ -13,7 +13,7 @@ // along with this program. If not, see // . -// +build darwin linux solaris freebsd +//go:build darwin || linux || solaris || freebsd package services