Merge branch 'main' into feature/check-on-gh-actions

.github/CODEOWNERS

@@ -19,7 +19,7 @@ CHANGELOG*
 /NOTICE.txt @elastic/beats-tech-leads
 
 /.ci/ @elastic/elastic-agent-data-plane @elastic/ingest-eng-prod
-/.github/ @elastic/elastic-agent-data-plane
+/.github/ @elastic/ingest-eng-prod
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
 /deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring

.github/dependabot.yml

@@ -38,3 +38,19 @@ updates:
  # Skip github.com/elastic/mito because it requires documentation updates.
  - dependency-name: github.com/elastic/mito
  open-pull-requests-limit: 2
+
+ # GitHub actions
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ day: "sunday"
+ time: "22:00"
+ labels:
+ - automation
+ - dependabot
+ groups:
+ github-actions:
+ patterns:
+ - "*"
+ open-pull-requests-limit: 5

.github/workflows/macos-xpack-heartbeat.yml

@@ -17,15 +17,17 @@ jobs:
  macos:
  runs-on: macos-latest
  steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-go@v3
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v4
  with:
  go-version-file: .go-version
  - uses: actions/setup-python@v4
  with:
  python-version: '3.9'
  - name: Install dependencies
- run: go install github.com/magefile/mage
+ run: |
+ go install github.com/magefile/mage
+ cd ${{ env.BEAT_MODULE }} && npm install -g @elastic/synthetics
  - name: Run build
  run: cd ${{ env.BEAT_MODULE }} && mage build
  - name: Run test

.go-version

@@ -1 +1 @@
-1.21.7
+1.21.8

.golangci.yml

@@ -114,7 +114,7 @@ linters-settings:
 
  gosimple:
  # Select the Go version to target. The default is '1.13'.
- go: "1.21.7"
+ go: "1.21.8"
 
  nakedret:
  # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -132,19 +132,19 @@ linters-settings:
 
  staticcheck:
  # Select the Go version to target. The default is '1.13'.
- go: "1.21.7"
+ go: "1.21.8"
  checks: ["all"]
 
  stylecheck:
  # Select the Go version to target. The default is '1.13'.
- go: "1.21.7"
+ go: "1.21.8"
  # Disabled:
  # ST1005: error strings should not be capitalized
  checks: ["all", "-ST1005"]
 
  unused:
  # Select the Go version to target. The default is '1.13'.
- go: "1.21.7"
+ go: "1.21.8"
 
  gosec:
  excludes:

CHANGELOG.next.asciidoc

@@ -10,7 +10,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
-- Upgrade to Go 1.21.7. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37913[37913]
+- Upgrade to Go 1.21.8. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}38209[38209]
 - add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack
 platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you
 know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`,
@@ -96,8 +96,10 @@ fields added to events containing the Beats version. {pull}37553[37553]
 - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799]
 - [threatintel] MISP pagination fixes {pull}37898[37898]
 - Fix file handle leak when handling errors in filestream {pull}37973[37973]
+- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094]
 - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116]
 - Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125]
+- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181]
 
 *Heartbeat*
 
@@ -107,6 +109,7 @@ fields added to events containing the Beats version. {pull}37553[37553]
 
 *Metricbeat*
 
+- Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720]
 
 *Osquerybeat*
 
@@ -137,8 +140,10 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d
 - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259]
 - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544]
 - Make more selective the Pod autodiscovery upon node and namespace update events. {issue}37338[37338] {pull}37431[37431]
+- Enhance add_cloud_metadata processor with `orchestrator.cluster.name`, `orchestrator.cluster.id` and `azure.resourcegroup.name` when running inside an AKS cluster. {issue}33081[33081] {pull}37685[37685]
 - Upgrade go-sysinfo from 1.12.0 to 1.13.1. {pull}37996[37996]
 - Make `range` condition work with numeric values as strings. {pull}38080[38080]
+- Allow users to configure number of output workers (for outputs that support workers) with either `worker` or `workers`. {pull}38257[38257]
 
 *Auditbeat*
 

NOTICE.txt

@@ -1773,11 +1773,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-sdk-for-go/sdk/azidentity
-Version: v1.2.2
+Version: v1.4.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/azidentity@v1.2.2/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/azidentity@v1.4.0/LICENSE.txt:
 
 MIT License
 
@@ -1832,6 +1832,36 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+--------------------------------------------------------------------------------
+Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4
+Version: v4.6.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4@v4.6.0/LICENSE.txt:
+
+MIT License
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/costmanagement/armcostmanagement
 Version: v1.1.0
@@ -1894,11 +1924,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources
-Version: v1.0.0
+Version: v1.1.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources@v1.0.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources@v1.1.1/LICENSE.txt:
 
 MIT License
 
@@ -13041,11 +13071,11 @@ these terms.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-system-metrics
-Version: v0.9.1
+Version: v0.9.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.1/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.2/LICENSE.txt:
 
  Apache License
  Version 2.0, January 2004
@@ -29549,6 +29579,98 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal
+Version: v1.1.2
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/internal@v1.1.2/LICENSE.txt:
+
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE
+
+
+--------------------------------------------------------------------------------
+Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2
+Version: v2.0.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2@v2.0.0/LICENSE.txt:
+
+MIT License
+
+Copyright (c) Microsoft Corporation.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE
+
+
+--------------------------------------------------------------------------------
+Dependency : github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups
+Version: v1.0.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups@v1.0.0/LICENSE.txt:
+
+MIT License
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-amqp
 Version: v0.16.0
@@ -31253,11 +31375,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/AzureAD/microsoft-authentication-library-for-go
-Version: v0.9.0
+Version: v1.1.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure!a!d/microsoft-authentication-library-for-go@v0.9.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure!a!d/microsoft-authentication-library-for-go@v1.1.1/LICENSE:
 
  MIT License
 
@@ -39522,12 +39644,12 @@ Contents of probable licence file $GOMODCACHE/github.com/godror/knownpb@v0.1.0/L
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/golang-jwt/jwt/v4
-Version: v4.5.0
+Dependency : github.com/golang-jwt/jwt/v5
+Version: v5.0.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/golang-jwt/jwt/v4@v4.5.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/golang-jwt/jwt/v5@v5.0.0/LICENSE:
 
 Copyright (c) 2012 Dave Grijalva
 Copyright (c) 2021 golang-jwt maintainers

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.7
+FROM golang:1.21.8
 
 RUN \
  apt-get update \

dev-tools/kubernetes/filebeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/heartbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

dev-tools/kubernetes/metricbeat/Dockerfile.debug

@@ -1,4 +1,4 @@
-FROM golang:1.21.7 as builder
+FROM golang:1.21.8 as builder
 
 ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin
 

go.mod

@@ -185,11 +185,12 @@ require (
  cloud.google.com/go/compute v1.23.0
  cloud.google.com/go/redis v1.13.1
  github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0
- github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2
+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
  github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0
+ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0
  github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/costmanagement/armcostmanagement v1.1.0
  github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.8.0
- github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.0.0
+ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1
  github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
  github.com/Azure/go-autorest/autorest/adal v0.9.14
  github.com/apache/arrow/go/v14 v14.0.2
@@ -204,7 +205,7 @@ require (
  github.com/elastic/elastic-agent-autodiscover v0.6.7
  github.com/elastic/elastic-agent-libs v0.7.5
  github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
- github.com/elastic/elastic-agent-system-metrics v0.9.1
+ github.com/elastic/elastic-agent-system-metrics v0.9.2
  github.com/elastic/go-elasticsearch/v8 v8.12.0
  github.com/elastic/mito v1.9.0
  github.com/elastic/tk-btf v0.1.0
@@ -249,7 +250,7 @@ require (
  github.com/Azure/go-autorest/logger v0.2.1 // indirect
  github.com/Azure/go-autorest/tracing v0.6.0 // indirect
  github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
- github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
+ github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
  github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
  github.com/andybalholm/brotli v1.0.5 // indirect
  github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
@@ -294,7 +295,7 @@ require (
  github.com/gobuffalo/here v0.6.7 // indirect
  github.com/goccy/go-json v0.10.2 // indirect
  github.com/godror/knownpb v0.1.0 // indirect
- github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+ github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
  github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
  github.com/golang-sql/sqlexp v0.1.0 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect