Skip to content

Commit

Permalink
Add ARM64 binary and package builds (#480)
Browse files Browse the repository at this point in the history
- Add ARM64 binary and package builds
- Download ARM64 beats and include them in the OS package
- Add filebeat and auditbeat to the dist tarball
- Cleanup Makefile
  - Put .PHONY directly in front of the targets
  - Add architecture names to each target
  - Remove no-op solaris patch download. We switched to elastic/gosigar
    and the patch is installed for cloudfoundry/gosigar.
  - Make build less verbose by removing the -v flag from go build
  - Make *-all targets more readable
- Fix solaris build
- Let fetch_collectors.sh fail early

Co-authored-by: Marco Pfatschbacher <marco@graylog.com>
  • Loading branch information
bernd and mpfz0r authored Oct 25, 2023
1 parent 192bf74 commit 87d003f
Show file tree
Hide file tree
Showing 9 changed files with 169 additions and 35 deletions.
109 changes: 79 additions & 30 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,133 +22,182 @@ WINDOWS_INSTALLER_VERSION = $(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)$(subst -,
# Removing the dot to comply with NuGet versioning (beta.1 -> beta2)
CHOCOLATEY_VERSION = $(COLLECTOR_VERSION).$(COLLECTOR_REVISION)$(subst .,,$(COLLECTOR_VERSION_SUFFIX))

.PHONY: all
all: build

.PHONY: fmt
fmt: ## Run gofmt
@GOFMT=$(GOFMT) sh ./format.sh

.PHONY: clean
clean: ## Remove binaries
-rm -rf $(targets)

.PHONY: distclean
distclean: clean
-rm -rf $(dist_targets)

.PHONY: test
test: ## Run tests
$(GO) test -v $(TEST_SUITE)

.PHONY: build
build: ## Build sidecar binary for local target system
$(GO) build $(BUILD_OPTS) -v -o graylog-sidecar
$(GO) build $(BUILD_OPTS) -o graylog-sidecar

build-all: build-linux-armv7 build-linux build-linux32 build-windows build-windows32 build-darwin build-darwin-arm64 build-freebsd
.PHONY: build-all
build-all: build-linux-amd64 build-linux-arm64 build-linux-armv7 build-linux32
build-all: build-darwin-amd64 build-darwin-arm64
build-all: build-freebsd-amd64
build-all: build-windows-amd64 build-windows32

build-linux: ## Build sidecar binary for Linux
.PHONY: build-linux-amd64
build-linux-amd64: ## Build sidecar binary for linux-amd64
@mkdir -p build/$(COLLECTOR_VERSION)/linux/amd64
GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar
GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar

solaris-sigar-patch:
# https://github.com/cloudfoundry/gosigar/pull/28
@if [ ! -e vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go ]; then \
wget -O vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go https://raw.githubusercontent.com/amitkris/gosigar/9fc0903125acd1a0dc7635f8670088339865bcd5/sigar_solaris.go; \
fi
.PHONY: build-linux-arm64
build-linux-arm64: ## Build sidecar binary for linux-arm64
@mkdir -p build/$(COLLECTOR_VERSION)/linux/arm64
GOOS=linux GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-arm64 -o build/$(COLLECTOR_VERSION)/linux/arm64/graylog-sidecar

.PHONY: build-linux-armv7
build-linux-armv7: ## Build sidecar binary for linux-armv7
@mkdir -p build/$(COLLECTOR_VERSION)/linux/armv7
GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7 -v -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar

build-solaris: solaris-sigar-patch ## Build sidecar binary for Solaris/OmniOS/Illumos
@mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64
GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar
GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7 -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar

.PHONY: build-linux32
build-linux32: ## Build sidecar binary for Linux 32bit
@mkdir -p build/$(COLLECTOR_VERSION)/linux/386
GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32 -v -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar
GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32 -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar

build-darwin: ## Build sidecar binary for OSX
.PHONY: build-darwin-amd64
build-darwin-amd64: ## Build sidecar binary for OSX
@mkdir -p build/$(COLLECTOR_VERSION)/darwin/amd64
GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -v -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar
GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar

.PHONY: build-darwin-arm64
build-darwin-arm64: ## Build sidecar binary for OSX
@mkdir -p build/$(COLLECTOR_VERSION)/darwin/arm64
GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -v -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar
GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar

build-freebsd: ## Build sidecar binary for FreeBSD
.PHONY: build-freebsd-amd64
build-freebsd-amd64: ## Build sidecar binary for FreeBSD
@mkdir -p build/$(COLLECTOR_VERSION)/freebsd/amd64
GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -v -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar
GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar

build-windows: install-goversioninfo ## Build sidecar binary for Windows
.PHONY: build-windows-amd64
build-windows-amd64: install-goversioninfo ## Build sidecar binary for Windows
@mkdir -p build/$(COLLECTOR_VERSION)/windows/amd64
$(GOVERSIONINFO_BIN) -64 -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso
GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -v -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe
GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe

.PHONY: build-windows32
build-windows32: install-goversioninfo ## Build sidecar binary for Windows 32bit
@mkdir -p build/$(COLLECTOR_VERSION)/windows/386
$(GOVERSIONINFO_BIN) -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso
GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -v -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe
GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe

.PHONY: build-solaris
build-solaris: ## Build sidecar binary for Solaris/OmniOS/Illumos
@mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64
GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar

.PHONY: sign-binaries
sign-binaries: sign-binary-windows-amd64 sign-binary-windows-386

.PHONY: sign-binary-windows-amd64
sign-binary-windows-amd64:
# This needs to run in a Docker container with the graylog/internal-codesigntool image
codesigntool sign build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe

.PHONY: sign-binary-windows-386
sign-binary-windows-386:
# This needs to run in a Docker container with the graylog/internal-codesigntool image
codesigntool sign build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe

## Adds version info to Windows executable
.PHONY: install-goversioninfo
install-goversioninfo:
go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest

package-all: prepare-package package-linux-armv7 package-linux package-linux32 package-windows package-tar
.PHONY: package-all
package-all: prepare-package
package-all: package-linux-armv7 package-linux-arm64 package-linux-amd64 package-linux32
package-all: package-windows-amd64
package-all: package-tar

.PHONY: prepare-package
prepare-package:
dist/fetch_collectors.sh

.PHONY: package-linux-armv7
package-linux-armv7: ## Create Linux ARMv7 system package
fpm-cook clean dist/recipearmv7.rb
rm -rf dist/cache dist/tmp-build dist/tmp-dest
fpm-cook -t deb package dist/recipearmv7.rb
fpm-cook -t rpm package dist/recipearmv7.rb

package-linux: ## Create Linux amd64 system package
.PHONY: package-linux-arm64
package-linux-arm64: ## Create Linux ARM64 system package
fpm-cook clean dist/recipearm64.rb
rm -rf dist/cache dist/tmp-build dist/tmp-dest
fpm-cook -t deb package dist/recipearm64.rb
fpm-cook -t rpm package dist/recipearm64.rb

.PHONY: package-linux-amd64
package-linux-amd64: ## Create Linux amd64 system package
fpm-cook clean dist/recipe.rb
rm -rf dist/cache dist/tmp-build dist/tmp-dest
fpm-cook -t deb package dist/recipe.rb
fpm-cook -t rpm package dist/recipe.rb

.PHONY: package-linux32
package-linux32: ## Create Linux i386 system package
fpm-cook clean dist/recipe32.rb
rm -rf dist/cache dist/tmp-build dist/tmp-dest
fpm-cook -t deb package dist/recipe32.rb
fpm-cook -t rpm package dist/recipe32.rb

package-windows: prepare-package ## Create Windows installer
.PHONY: package-windows-amd64
package-windows-amd64: prepare-package ## Create Windows installer
@mkdir -p dist/pkg
makensis -DVERSION=$(COLLECTOR_VERSION) -DVERSION_SUFFIX=$(COLLECTOR_VERSION_SUFFIX) -DREVISION=$(COLLECTOR_REVISION) dist/recipe.nsi

.PHONY: sign-windows-installer
sign-windows-installer:
# This needs to run in a Docker container with the graylog/internal-codesigntool image
codesigntool sign dist/pkg/graylog_sidecar_installer_$(WINDOWS_INSTALLER_VERSION).exe

.PHONY: package-chocolatey
package-chocolatey: ## Create Chocolatey .nupkg file
# This needs to run in a Docker container based on the Dockerfile.chocolatey image!
dist/chocolatey/gensha.sh $(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX) $(WINDOWS_INSTALLER_VERSION)
# The fourth number in Chocolatey (NuGet) is the revision.
# See: https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#where-nugetversion-diverges-from-semantic-versioning
cd dist/chocolatey && choco pack graylog-sidecar.nuspec --version $(CHOCOLATEY_VERSION) --out ../pkg

.PHONY: push-chocolatey
push-chocolatey: ## Push Chocolatey .nupkg file
# This needs to run in a Docker container based on the Dockerfile.chocolatey image!
# Escape the CHOCO_API_KEY to avoid printing it in the logs!
choco push dist/pkg/graylog-sidecar.$(CHOCOLATEY_VERSION).nupkg -k=$$CHOCO_API_KEY

.PHONY: package-tar
package-tar: ## Create tar archive for all platforms
@mkdir -p dist/pkg
@tar --transform="s|/build|/graylog-sidecar|" -Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz ./build ./sidecar-example.yml ./sidecar-windows-example.yml

@tar --transform="s|/build|/graylog-sidecar|" --transform="s|/dist|/graylog-sidecar|" \
-Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz \
./build \
./dist/collectors/auditbeat/linux/arm64/auditbeat \
./dist/collectors/auditbeat/linux/x86_64/auditbeat \
./dist/collectors/filebeat/linux/arm64/filebeat \
./dist/collectors/filebeat/linux/x86_64/filebeat \
./sidecar-example.yml \
./sidecar-windows-example.yml

.PHONY: help
help:
@grep -hE '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | $(AWK) 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

.DEFAULT_GOAL := all

.PHONY: all build build-all build-linux build-linux32 build-darwin build-freebsd build-windows build-windows32 fmt clean distclean help package-all package-linux package-linux32 package-windows package-tar
15 changes: 15 additions & 0 deletions api/version.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright (C) 2020 Graylog, Inc.

// This program is free software: you can redistribute it and/or modify
// it under the terms of the Server Side Public License, version 1,
// as published by MongoDB, Inc.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// Server Side Public License for more details.
//
// You should have received a copy of the Server Side Public License
// along with this program. If not, see
// <http://www.mongodb.com/licensing/server-side-public-license>.

package api

import "github.com/hashicorp/go-version"
Expand Down
4 changes: 4 additions & 0 deletions changelog/unreleased/pr-480.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
type = "a"
message = "Add operating system packages for Linux ARM64."

pulls = ["480"]
2 changes: 1 addition & 1 deletion common/sigar.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// along with this program. If not, see
// <http://www.mongodb.com/licensing/server-side-public-license>.

// +build !freebsd,!darwin
//go:build !freebsd && !darwin && !solaris

package common

Expand Down
28 changes: 28 additions & 0 deletions common/sigar_solaris.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright (C) 2020 Graylog, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the Server Side Public License, version 1,
// as published by MongoDB, Inc.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// Server Side Public License for more details.
//
// You should have received a copy of the Server Side Public License
// along with this program. If not, see
// <http://www.mongodb.com/licensing/server-side-public-license>.

package common

func GetCpuIdle() float64 {
return -1
}

func GetFileSystemList75(string) []string {
return []string{}
}

func GetLoad1() float64 {
return -1
}
2 changes: 1 addition & 1 deletion daemon/svc_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// along with this program. If not, see
// <http://www.mongodb.com/licensing/server-side-public-license>.

// +build !windows
//go:build !windows

package daemon

Expand Down
8 changes: 6 additions & 2 deletions dist/fetch_collectors.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

set -eo pipefail

FILEBEAT_VERSION=8.9.0
FILEBEAT_VERSION_32=7.17.12
WINLOGBEAT_VERSION=8.9.0
Expand All @@ -24,7 +26,7 @@ download_beat()
archive="/tmp/${name}-${version}-${os}-${arch}.zip"
if [ ! -f $archive ]; then
echo "==> Downloading ${name}-${version}-${os}-${arch}"
curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip
curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip
fi
unzip -o -d dist/collectors/${name}/${os}/${arch} $archive
mv dist/collectors/${name}/${os}/${arch}/${name}-${version}-${os}-${arch}/* dist/collectors/${name}/${os}/${arch}/
Expand All @@ -34,7 +36,7 @@ download_beat()
archive="/tmp/${name}-${version}-${os}-${arch}.tar.gz"
if [ ! -f $archive ]; then
echo "==> Downloading ${name}-${version}-${os}-${arch}"
curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz
curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz
fi
tar -xzf $archive --strip-components=1 -C dist/collectors/${name}/${os}/${arch}
;;
Expand All @@ -43,9 +45,11 @@ download_beat()

download_beat "filebeat" "linux" ${FILEBEAT_VERSION} x86_64
download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} x86
download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} arm64

download_beat "auditbeat" "linux" ${FILEBEAT_VERSION} x86_64
download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} x86
download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} arm64

download_beat "filebeat" "windows" ${FILEBEAT_VERSION} x86_64
download_beat "filebeat" "windows" ${FILEBEAT_VERSION_32} x86
Expand Down
34 changes: 34 additions & 0 deletions dist/recipearm64.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
require_relative 'tools'

class GraylogSidecar < FPM::Cookery::Recipe
description 'Graylog collector sidecar'

name 'graylog-sidecar'
version data.version
revision data.revision
homepage 'https://graylog.org'
arch 'arm64'

source "file:../../build/#{version}/linux/arm64/graylog-sidecar"

maintainer 'Graylog, Inc. <hello@graylog.org>'
vendor 'graylog'
license 'SSPL'

config_files '/etc/graylog/sidecar/sidecar.yml'

fpm_attributes rpm_os: 'linux'

def build
end

def install
bin.install 'graylog-sidecar'
lib('graylog-sidecar').install '../../collectors/filebeat/linux/arm64/filebeat'
lib('graylog-sidecar').install '../../collectors/auditbeat/linux/arm64/auditbeat'
etc('graylog/sidecar').install '../../../sidecar-example.yml', 'sidecar.yml'
var('lib/graylog-sidecar/generated').mkdir
var('log/graylog-sidecar').mkdir
var('run/graylog-sidecar').mkdir
end
end
2 changes: 1 addition & 1 deletion services/control_handler_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// along with this program. If not, see
// <http://www.mongodb.com/licensing/server-side-public-license>.

// +build darwin linux solaris freebsd
//go:build darwin || linux || solaris || freebsd

package services

Expand Down

0 comments on commit 87d003f

Please sign in to comment.