Merge remote-tracking branch 'origin/master' into update-go-and-deps

Makefile

@@ -22,133 +22,182 @@ WINDOWS_INSTALLER_VERSION = $(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)$(subst -,
 # Removing the dot to comply with NuGet versioning (beta.1 -> beta2)
 CHOCOLATEY_VERSION = $(COLLECTOR_VERSION).$(COLLECTOR_REVISION)$(subst .,,$(COLLECTOR_VERSION_SUFFIX))
 
+.PHONY: all
 all: build
 
+.PHONY: fmt
 fmt: ## Run gofmt
 	@GOFMT=$(GOFMT) sh ./format.sh
 
+.PHONY: clean
 clean: ## Remove binaries
 	-rm -rf $(targets)
 
+.PHONY: distclean
 distclean: clean
 	-rm -rf $(dist_targets)
 
+.PHONY: test
 test: ## Run tests
 	$(GO) test -v $(TEST_SUITE)
 
+.PHONY: build
 build: ## Build sidecar binary for local target system
-	$(GO) build $(BUILD_OPTS) -v -o graylog-sidecar
+	$(GO) build $(BUILD_OPTS) -o graylog-sidecar
 
-build-all: build-linux-armv7 build-linux build-linux32 build-windows build-windows32 build-darwin build-darwin-arm64 build-freebsd
+.PHONY: build-all
+build-all: build-linux-amd64 build-linux-arm64 build-linux-armv7 build-linux32
+build-all: build-darwin-amd64 build-darwin-arm64
+build-all: build-freebsd-amd64
+build-all: build-windows-amd64 build-windows32
 
-build-linux: ## Build sidecar binary for Linux
+.PHONY: build-linux-amd64
+build-linux-amd64: ## Build sidecar binary for linux-amd64
 	@mkdir -p build/$(COLLECTOR_VERSION)/linux/amd64
-	GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar
+	GOOS=linux GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/linux/amd64/graylog-sidecar
 
-solaris-sigar-patch:
-	# https://github.com/cloudfoundry/gosigar/pull/28
-	@if [ ! -e vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go ]; then \
-		wget -O vendor/github.com/cloudfoundry/gosigar/sigar_solaris.go https://raw.githubusercontent.com/amitkris/gosigar/9fc0903125acd1a0dc7635f8670088339865bcd5/sigar_solaris.go; \
-	fi
+.PHONY: build-linux-arm64
+build-linux-arm64: ## Build sidecar binary for linux-arm64
+	@mkdir -p build/$(COLLECTOR_VERSION)/linux/arm64
+	GOOS=linux GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-arm64  -o build/$(COLLECTOR_VERSION)/linux/arm64/graylog-sidecar
 
+.PHONY: build-linux-armv7
 build-linux-armv7: ## Build sidecar binary for linux-armv7
 	@mkdir -p build/$(COLLECTOR_VERSION)/linux/armv7
-	GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7  -v -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar
-
-build-solaris: solaris-sigar-patch ## Build sidecar binary for Solaris/OmniOS/Illumos
-	@mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64
-	GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -v -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar
+	GOOS=linux GOARCH=arm GOARM=7 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux-armv7  -o build/$(COLLECTOR_VERSION)/linux/armv7/graylog-sidecar
 
+.PHONY: build-linux32
 build-linux32: ## Build sidecar binary for Linux 32bit
 	@mkdir -p build/$(COLLECTOR_VERSION)/linux/386
-	GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32  -v -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar
+	GOOS=linux GOARCH=386 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_linux32  -o build/$(COLLECTOR_VERSION)/linux/386/graylog-sidecar
 
-build-darwin: ## Build sidecar binary for OSX
+.PHONY: build-darwin-amd64
+build-darwin-amd64: ## Build sidecar binary for OSX
 	@mkdir -p build/$(COLLECTOR_VERSION)/darwin/amd64
-	GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -v -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar
+	GOOS=darwin GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin -o build/$(COLLECTOR_VERSION)/darwin/amd64/graylog-sidecar
 
+.PHONY: build-darwin-arm64
 build-darwin-arm64: ## Build sidecar binary for OSX
 	@mkdir -p build/$(COLLECTOR_VERSION)/darwin/arm64
-	GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -v -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar
+	GOOS=darwin GOARCH=arm64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_darwin-arm64 -o build/$(COLLECTOR_VERSION)/darwin/arm64/graylog-sidecar
 
-build-freebsd: ## Build sidecar binary for FreeBSD
+.PHONY: build-freebsd-amd64
+build-freebsd-amd64: ## Build sidecar binary for FreeBSD
 	@mkdir -p build/$(COLLECTOR_VERSION)/freebsd/amd64
-	GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -v -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar
+	GOOS=freebsd GOARCH=amd64 $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_freebsd -o build/$(COLLECTOR_VERSION)/freebsd/amd64/graylog-sidecar
 
-build-windows: install-goversioninfo ## Build sidecar binary for Windows
+.PHONY: build-windows-amd64
+build-windows-amd64: install-goversioninfo ## Build sidecar binary for Windows
 	@mkdir -p build/$(COLLECTOR_VERSION)/windows/amd64
 	$(GOVERSIONINFO_BIN) -64 -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso
-	GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -v -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe
+	GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win -o build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe
 
+.PHONY: build-windows32
 build-windows32: install-goversioninfo ## Build sidecar binary for Windows 32bit
 	@mkdir -p build/$(COLLECTOR_VERSION)/windows/386
 	$(GOVERSIONINFO_BIN) -product-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -product-ver-minor="$(COLLECTOR_VERSION_MINOR)" -product-ver-patch="$(COLLECTOR_VERSION_PATCH)" -product-ver-build="$(COLLECTOR_REVISION)" -file-version="$(COLLECTOR_VERSION)-$(COLLECTOR_REVISION)" -ver-major="$(COLLECTOR_VERSION_MAJOR)" -ver-minor="$(COLLECTOR_VERSION_MINOR)" -ver-patch="$(COLLECTOR_VERSION_PATCH)" -ver-build="$(COLLECTOR_REVISION)" -o resource_windows.syso
-	GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -v -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe
+	GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc $(GO) build $(BUILD_OPTS) -pkgdir $(GOPATH)/go_win32 -o build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe
+
+.PHONY: build-solaris
+build-solaris: ## Build sidecar binary for Solaris/OmniOS/Illumos
+	@mkdir -p build/$(COLLECTOR_VERSION)/solaris/amd64
+	GOOS=solaris GOARCH=amd64 $(GO) build $(BUILD_OPTS) -o build/$(COLLECTOR_VERSION)/solaris/amd64/graylog-sidecar
 
+.PHONY: sign-binaries
 sign-binaries: sign-binary-windows-amd64 sign-binary-windows-386
 
+.PHONY: sign-binary-windows-amd64
 sign-binary-windows-amd64:
 	# This needs to run in a Docker container with the graylog/internal-codesigntool image
 	codesigntool sign build/$(COLLECTOR_VERSION)/windows/amd64/graylog-sidecar.exe
 
+.PHONY: sign-binary-windows-386
 sign-binary-windows-386:
 	# This needs to run in a Docker container with the graylog/internal-codesigntool image
 	codesigntool sign build/$(COLLECTOR_VERSION)/windows/386/graylog-sidecar.exe
 
 ## Adds version info to Windows executable
+.PHONY: install-goversioninfo
 install-goversioninfo:
 	go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
 
-package-all: prepare-package package-linux-armv7 package-linux package-linux32 package-windows package-tar
+.PHONY: package-all
+package-all: prepare-package
+package-all: package-linux-armv7 package-linux-arm64 package-linux-amd64 package-linux32
+package-all: package-windows-amd64
+package-all: package-tar
 
+.PHONY: prepare-package
 prepare-package:
 	dist/fetch_collectors.sh
 
+.PHONY: package-linux-armv7
 package-linux-armv7: ## Create Linux ARMv7 system package
 	fpm-cook clean dist/recipearmv7.rb
 	rm -rf dist/cache dist/tmp-build dist/tmp-dest
 	fpm-cook -t deb package dist/recipearmv7.rb
 	fpm-cook -t rpm package dist/recipearmv7.rb
 
-package-linux: ## Create Linux amd64 system package
+.PHONY: package-linux-arm64
+package-linux-arm64: ## Create Linux ARM64 system package
+	fpm-cook clean dist/recipearm64.rb
+	rm -rf dist/cache dist/tmp-build dist/tmp-dest
+	fpm-cook -t deb package dist/recipearm64.rb
+	fpm-cook -t rpm package dist/recipearm64.rb
+
+.PHONY: package-linux-amd64
+package-linux-amd64: ## Create Linux amd64 system package
 	fpm-cook clean dist/recipe.rb
 	rm -rf dist/cache dist/tmp-build dist/tmp-dest
 	fpm-cook -t deb package dist/recipe.rb
 	fpm-cook -t rpm package dist/recipe.rb
 
+.PHONY: package-linux32
 package-linux32: ## Create Linux i386 system package
 	fpm-cook clean dist/recipe32.rb
 	rm -rf dist/cache dist/tmp-build dist/tmp-dest
 	fpm-cook -t deb package dist/recipe32.rb
 	fpm-cook -t rpm package dist/recipe32.rb
 
-package-windows: prepare-package ## Create Windows installer
+.PHONY: package-windows-amd64
+package-windows-amd64: prepare-package ## Create Windows installer
 	@mkdir -p dist/pkg
 	makensis -DVERSION=$(COLLECTOR_VERSION) -DVERSION_SUFFIX=$(COLLECTOR_VERSION_SUFFIX) -DREVISION=$(COLLECTOR_REVISION) dist/recipe.nsi
 
+.PHONY: sign-windows-installer
 sign-windows-installer:
 	# This needs to run in a Docker container with the graylog/internal-codesigntool image
 	codesigntool sign dist/pkg/graylog_sidecar_installer_$(WINDOWS_INSTALLER_VERSION).exe
 
+.PHONY: package-chocolatey
 package-chocolatey: ## Create Chocolatey .nupkg file
 	# This needs to run in a Docker container based on the Dockerfile.chocolatey image!
 	dist/chocolatey/gensha.sh $(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX) $(WINDOWS_INSTALLER_VERSION)
 	# The fourth number in Chocolatey (NuGet) is the revision.
 	# See: https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#where-nugetversion-diverges-from-semantic-versioning
 	cd dist/chocolatey && choco pack graylog-sidecar.nuspec --version $(CHOCOLATEY_VERSION) --out ../pkg
 
+.PHONY: push-chocolatey
 push-chocolatey: ## Push Chocolatey .nupkg file
 	# This needs to run in a Docker container based on the Dockerfile.chocolatey image!
 	# Escape the CHOCO_API_KEY to avoid printing it in the logs!
 	choco push dist/pkg/graylog-sidecar.$(CHOCOLATEY_VERSION).nupkg -k=$$CHOCO_API_KEY
 
+.PHONY: package-tar
 package-tar: ## Create tar archive for all platforms
 	@mkdir -p dist/pkg
-	@tar --transform="s|/build|/graylog-sidecar|" -Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz ./build ./sidecar-example.yml ./sidecar-windows-example.yml
-
+	@tar --transform="s|/build|/graylog-sidecar|" --transform="s|/dist|/graylog-sidecar|" \
+		-Pczf dist/pkg/graylog-sidecar-$(COLLECTOR_VERSION)$(COLLECTOR_VERSION_SUFFIX).tar.gz \
+		./build \
+		./dist/collectors/auditbeat/linux/arm64/auditbeat \
+		./dist/collectors/auditbeat/linux/x86_64/auditbeat \
+		./dist/collectors/filebeat/linux/arm64/filebeat \
+		./dist/collectors/filebeat/linux/x86_64/filebeat \
+		./sidecar-example.yml \
+		./sidecar-windows-example.yml
+
+.PHONY: help
 help:
 	@grep -hE '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | $(AWK) 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 .DEFAULT_GOAL := all
-
-.PHONY: all build build-all build-linux build-linux32 build-darwin build-freebsd build-windows build-windows32 fmt clean distclean help package-all package-linux package-linux32 package-windows package-tar

api/version.go

@@ -1,3 +1,18 @@
+// Copyright (C) 2020 Graylog, Inc.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the Server Side Public License, version 1,
+// as published by MongoDB, Inc.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// Server Side Public License for more details.
+//
+// You should have received a copy of the Server Side Public License
+// along with this program. If not, see
+// <http://www.mongodb.com/licensing/server-side-public-license>.
+
 package api
 
 import "github.com/hashicorp/go-version"

changelog/unreleased/pr-480.toml

@@ -0,0 +1,4 @@
+type = "a"
+message = "Add operating system packages for Linux ARM64."
+
+pulls = ["480"]

common/sigar.go

@@ -13,7 +13,7 @@
 // along with this program. If not, see
 // <http://www.mongodb.com/licensing/server-side-public-license>.
 
-// +build !freebsd,!darwin
+//go:build !freebsd && !darwin && !solaris
 
 package common
 

common/sigar_solaris.go

@@ -0,0 +1,28 @@
+// Copyright (C) 2020 Graylog, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the Server Side Public License, version 1,
+// as published by MongoDB, Inc.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// Server Side Public License for more details.
+//
+// You should have received a copy of the Server Side Public License
+// along with this program. If not, see
+// <http://www.mongodb.com/licensing/server-side-public-license>.
+
+package common
+
+func GetCpuIdle() float64 {
+	return -1
+}
+
+func GetFileSystemList75(string) []string {
+	return []string{}
+}
+
+func GetLoad1() float64 {
+	return -1
+}

daemon/svc_helper.go

@@ -13,7 +13,7 @@
 // along with this program. If not, see
 // <http://www.mongodb.com/licensing/server-side-public-license>.
 
-// +build !windows
+//go:build !windows
 
 package daemon
 

dist/fetch_collectors.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -eo pipefail
+
 FILEBEAT_VERSION=8.9.0
 FILEBEAT_VERSION_32=7.17.12
 WINLOGBEAT_VERSION=8.9.0
@@ -24,7 +26,7 @@ download_beat()
     archive="/tmp/${name}-${version}-${os}-${arch}.zip"
     if [ ! -f $archive ]; then
       echo "==> Downloading ${name}-${version}-${os}-${arch}"
-      curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip
+      curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.zip
     fi
     unzip -o -d dist/collectors/${name}/${os}/${arch} $archive
     mv dist/collectors/${name}/${os}/${arch}/${name}-${version}-${os}-${arch}/* dist/collectors/${name}/${os}/${arch}/
@@ -34,7 +36,7 @@ download_beat()
     archive="/tmp/${name}-${version}-${os}-${arch}.tar.gz"
     if [ ! -f $archive ]; then
       echo "==> Downloading ${name}-${version}-${os}-${arch}"
-      curl -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz
+      curl -fsSL -o $archive https://artifacts.elastic.co/downloads/beats/${name}/${name}-oss-${version}-${os}-${arch}.tar.gz
     fi
     tar -xzf $archive --strip-components=1 -C dist/collectors/${name}/${os}/${arch}
     ;;
@@ -43,9 +45,11 @@ download_beat()
 
 download_beat "filebeat" "linux" ${FILEBEAT_VERSION} x86_64
 download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} x86
+download_beat "filebeat" "linux" ${FILEBEAT_VERSION_32} arm64
 
 download_beat "auditbeat" "linux" ${FILEBEAT_VERSION} x86_64
 download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} x86
+download_beat "auditbeat" "linux" ${AUDITBEAT_VERSION_32} arm64
 
 download_beat "filebeat" "windows" ${FILEBEAT_VERSION} x86_64
 download_beat "filebeat" "windows" ${FILEBEAT_VERSION_32} x86

dist/recipearm64.rb

@@ -0,0 +1,34 @@
+require_relative 'tools'
+
+class GraylogSidecar < FPM::Cookery::Recipe
+  description 'Graylog collector sidecar'
+
+  name     'graylog-sidecar'
+  version  data.version
+  revision data.revision
+  homepage 'https://graylog.org'
+  arch     'arm64'
+
+  source   "file:../../build/#{version}/linux/arm64/graylog-sidecar"
+
+  maintainer 'Graylog, Inc. <hello@graylog.org>'
+  vendor     'graylog'
+  license    'SSPL'
+
+  config_files '/etc/graylog/sidecar/sidecar.yml'
+
+  fpm_attributes rpm_os: 'linux'
+
+  def build
+  end
+
+  def install
+    bin.install 'graylog-sidecar'
+    lib('graylog-sidecar').install '../../collectors/filebeat/linux/arm64/filebeat'
+    lib('graylog-sidecar').install '../../collectors/auditbeat/linux/arm64/auditbeat'
+    etc('graylog/sidecar').install '../../../sidecar-example.yml', 'sidecar.yml'
+    var('lib/graylog-sidecar/generated').mkdir
+    var('log/graylog-sidecar').mkdir
+    var('run/graylog-sidecar').mkdir
+  end
+end

services/control_handler_unix.go

@@ -13,7 +13,7 @@
 // along with this program. If not, see
 // <http://www.mongodb.com/licensing/server-side-public-license>.
 
-// +build darwin linux solaris freebsd
+//go:build darwin || linux || solaris || freebsd
 
 package services