-
Notifications
You must be signed in to change notification settings - Fork 360
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Gracefully drain listeners before envoy shutdown on pod termina…
…tion (#2633) * feat: Gracefully drain listeners before envoy shutdown on pod termination Signed-off-by: David Alger <davidmalger@gmail.com> * Setup hooks to manage graceful Envoy shutdown process Signed-off-by: David Alger <davidmalger@gmail.com> * Implement graceful drain process in shutdown-manager Signed-off-by: David Alger <davidmalger@gmail.com> * Send logs from exec prestop hook to stdout of main process Signed-off-by: David Alger <davidmalger@gmail.com> * Make linter happy Signed-off-by: David Alger <davidmalger@gmail.com> * Minor cleanup Signed-off-by: David Alger <davidmalger@gmail.com> * Stop polling when ready-timeout exceeded Signed-off-by: David Alger <davidmalger@gmail.com> * Container configuration for shutdown manager Signed-off-by: David Alger <davidmalger@gmail.com> * Setup health probes Signed-off-by: David Alger <davidmalger@gmail.com> * Configurable shutdown timeouts Signed-off-by: David Alger <davidmalger@gmail.com> * Correct exitAtConnections logic Signed-off-by: David Alger <davidmalger@gmail.com> * Lower shutdown timeouts for conformance tests Signed-off-by: David Alger <davidmalger@gmail.com> * Integrate with latest from main Signed-off-by: David Alger <davidmalger@gmail.com> * Describe node used in test runs Signed-off-by: David Alger <davidmalger@gmail.com> * Use TAG=latest for conformance tests Signed-off-by: David Alger <davidmalger@gmail.com> * Update shutdown/ready logic and misc cleanup Signed-off-by: David Alger <davidmalger@gmail.com> * Shutdown manager config test Signed-off-by: David Alger <davidmalger@gmail.com> * Test coverage FileLogger Signed-off-by: David Alger <davidmalger@gmail.com> * Require use of patch field to override config on shutdown-manager container Signed-off-by: David Alger <davidmalger@gmail.com> * Update docs Signed-off-by: David Alger <davidmalger@gmail.com> * Remove knob for ExitAtConnections Signed-off-by: David Alger <davidmalger@gmail.com> * Pass image version for shutdown-manager in from build Signed-off-by: David Alger <davidmalger@gmail.com> * Fix generated content Signed-off-by: David Alger <davidmalger@gmail.com> * Recombine image consts Signed-off-by: David Alger <davidmalger@gmail.com> * Lower default min-drain-duration to 5 seconds Signed-off-by: David Alger <davidmalger@gmail.com> * Update generated docs Signed-off-by: David Alger <davidmalger@gmail.com> * Fail health checks to support fast failure when active health checking Signed-off-by: David Alger <davidmalger@gmail.com> * Update tests Signed-off-by: David Alger <davidmalger@gmail.com> * Move drain type config to correct place Signed-off-by: David Alger <davidmalger@gmail.com> --------- Signed-off-by: David Alger <davidmalger@gmail.com>
- Loading branch information
1 parent
72fadb7
commit 329aafc
Showing
123 changed files
with
2,002 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| // Copyright Envoy Gateway Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // The full text of the Apache license is available in the LICENSE file at | ||
| // the root of the repo. | ||
|
|
||
| package cmd | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/envoyproxy/gateway/internal/cmd/envoy" | ||
| ) | ||
|
|
||
| // getEnvoyCommand returns the envoy cobra command to be executed. | ||
| func getEnvoyCommand() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "envoy", | ||
| Short: "Envoy proxy management", | ||
| } | ||
|
|
||
| cmd.AddCommand(getShutdownCommand()) | ||
| cmd.AddCommand(getShutdownManagerCommand()) | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| // getShutdownCommand returns the shutdown cobra command to be executed. | ||
| func getShutdownCommand() *cobra.Command { | ||
| var drainTimeout time.Duration | ||
| var minDrainDuration time.Duration | ||
| var exitAtConnections int | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "shutdown", | ||
| Short: "Gracefully drain open connections prior to pod shutdown.", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| return envoy.Shutdown(drainTimeout, minDrainDuration, exitAtConnections) | ||
| }, | ||
| } | ||
|
|
||
| cmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 600*time.Second, | ||
| "Graceful shutdown timeout. This should be less than the pod's terminationGracePeriodSeconds.") | ||
|
|
||
| cmd.PersistentFlags().DurationVar(&minDrainDuration, "min-drain-duration", 5*time.Second, | ||
| "Minimum drain duration allowing time for endpoint deprogramming to complete.") | ||
|
|
||
| cmd.PersistentFlags().IntVar(&exitAtConnections, "exit-at-connections", 0, | ||
| "Number of connections to wait for when monitoring Envoy listener drain process.") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| // getShutdownManagerCommand returns the shutdown manager cobra command to be executed. | ||
| func getShutdownManagerCommand() *cobra.Command { | ||
| var readyTimeout time.Duration | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "shutdown-manager", | ||
| Short: "Provides HTTP endpoint used in preStop hook to block until ready for pod shutdown.", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| return envoy.ShutdownManager(readyTimeout) | ||
| }, | ||
| } | ||
|
|
||
| cmd.PersistentFlags().DurationVar(&readyTimeout, "ready-timeout", 610*time.Second, | ||
| "Shutdown ready timeout. This should be greater than shutdown's drain-timeout and less than the pod's terminationGracePeriodSeconds.") | ||
|
|
||
| return cmd | ||
| } |
Oops, something went wrong.