Skip to content

Commit

Permalink
[supervisor] revert recent changes to reduce frequency of `supervisor…
Browse files Browse the repository at this point in the history
… run error with unexpected exit code` (#20325)

* Revert "[supervisor] switch lib back to use upstream `ramr/go-reaper` (#20322)"

This reverts commit 9442b52.

* Revert "[supervisor] set pod failure reason when supervisor is reaped (#20318)"

This reverts commit 9e8da3b.
  • Loading branch information
kylos101 authored Oct 29, 2024
1 parent 6bc053e commit 3f066ce
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 48 deletions.
50 changes: 8 additions & 42 deletions components/supervisor/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,59 +77,25 @@ var initCmd = &cobra.Command{
}

supervisorDone := make(chan struct{})
handledByReaper := make(chan int)
handleSupervisorExit := func(exitCode int) {
if exitCode == 0 {
return
}
logs := extractFailureFromRun()
if shared.IsExpectedShutdown(exitCode) {
log.Fatal(logs)
} else {
log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
}
}
go func() {
defer close(supervisorDone)

err := runCommand.Wait()
if err == nil {
return
}
// exited by reaper
if strings.Contains(err.Error(), "no child processes") {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
select {
case <-ctx.Done(): // timeout
case exitCode := <-handledByReaper:
handleSupervisorExit(exitCode)
}
} else if !(strings.Contains(err.Error(), "signal: ")) {
if err != nil && !(strings.Contains(err.Error(), "signal: ") || strings.Contains(err.Error(), "no child processes")) {
if eerr, ok := err.(*exec.ExitError); ok && eerr.ExitCode() != 0 {
handleSupervisorExit(eerr.ExitCode())
logs := extractFailureFromRun()
if shared.IsExpectedShutdown(eerr.ExitCode()) {
log.Fatal(logs)
} else {
log.WithError(fmt.Errorf(logs)).Fatal("supervisor run error with unexpected exit code")
}
}
log.WithError(err).Error("supervisor run error")
return
}
}()
// start the reaper to clean up zombie processes
reaperChan := make(chan reaper.Status, 10)
reaper.Start(reaper.Config{
Pid: -1,
Options: 0,
DisablePid1Check: false,
StatusChannel: reaperChan,
})
go func() {
for status := range reaperChan {
if status.Pid != runCommand.Process.Pid {
continue
}
exitCode := status.WaitStatus.ExitStatus()
handledByReaper <- exitCode
}
}()
reaper.Reap()

select {
case <-supervisorDone:
Expand Down
3 changes: 1 addition & 2 deletions components/supervisor/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ require (
github.com/gitpod-io/gitpod/ide-metrics-api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/supervisor/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/go-reaper v0.0.0-20241024192051-78d04cc2e25f
github.com/golang/mock v1.6.0
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
Expand All @@ -30,7 +29,7 @@ require (
github.com/prometheus/common v0.42.0
github.com/prometheus/procfs v0.10.1
github.com/prometheus/pushgateway v1.5.1
github.com/ramr/go-reaper v0.2.2
github.com/ramr/go-reaper v0.2.1
github.com/sirupsen/logrus v1.9.3
github.com/soheilhy/cmux v0.1.5
github.com/spf13/cobra v1.4.0
Expand Down
6 changes: 2 additions & 4 deletions components/supervisor/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 3f066ce

Please sign in to comment.