Getting Fatal Memory error in Wave Server

[n-srinidhi]

Hi!I have a Wave app deployed into production, Once the wave server is running, a few hours later I get a Memory exception with the following stacktrace on the command prompt:

fatal error: out of memory

runtime stack:
runtime.throw({0xd6180f?, 0xc0000a1000?})
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/panic.go:1047 +0x65 fp=0xd2e39ff3e0 sp=0xd2e39ff3b0 pc=0x9489c5
runtime.sysUsedOS(0xc00009c000, 0x8000)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mem_windows.go:83 +0x1c9 fp=0xd2e39ff440 sp=0xd2e39ff3e0 pc=0x928b89
runtime.sysUsed(0x10838f0?, 0x4?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mem.go:77 +0x25 fp=0xd2e39ff460 sp=0xd2e39ff440 pc=0x928685
runtime.(*mheap).allocSpan(0x10838e0, 0x4, 0x3, 0x0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mheap.go:1342 +0x619 fp=0xd2e39ff4f8 sp=0xd2e39ff460 pc=0x9390f9
runtime.(*mheap).allocManual(0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mheap.go:944 +0x1f fp=0xd2e39ff520 sp=0xd2e39ff4f8 pc=0x93871f
runtime.getempty.func1()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcwork.go:377 +0x33 fp=0xd2e39ff550 sp=0xd2e39ff520 pc=0x937613
runtime.getempty()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcwork.go:376 +0xcc fp=0xd2e39ff5b0 sp=0xd2e39ff550 pc=0x93748c
runtime.(*gcWork).init(0xc000039240)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcwork.go:100 +0x1c fp=0xd2e39ff5c0 sp=0xd2e39ff5b0 pc=0x936cfc
runtime.(*gcWork).put(0xc000039240, 0xc000130000)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcwork.go:120 +0x95 fp=0xd2e39ff5e8 sp=0xd2e39ff5c0 pc=0x936dd5
runtime.greyobject(0xc000130000, 0xc00005ea78?, 0x975c0d?, 0xc00003a500?, 0x0?, 0xd2e39ff690?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcmark.go:1509 +0x1a5 fp=0xd2e39ff638 sp=0xd2e39ff5e8 pc=0x930f05
runtime.scanblock(0x10692c0, 0x40000, 0x2cbebd2d375, 0xc0000851e0?, 0x0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcmark.go:1238 +0x114 fp=0xd2e39ff6a0 sp=0xd2e39ff638 pc=0x9306b4
runtime.markrootBlock(0xd2e39ff708?, 0x944dca?, 0x7ff8c68e4310?, 0x15c?, 0x1?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcmark.go:285 +0x65 fp=0xd2e39ff6e0 sp=0xd2e39ff6a0 pc=0x92e705
runtime.markroot(0xc000039240, 0x3, 0x1)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcmark.go:176 +0x367 fp=0xd2e39ff780 sp=0xd2e39ff6e0 pc=0x92e447
runtime.gcDrain(0xc000039240, 0x3)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcmark.go:1069 +0x39f fp=0xd2e39ff7e0 sp=0xd2e39ff780 pc=0x9302df
runtime.gcBgMarkWorker.func2()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1308 +0xa5 fp=0xd2e39ff830 sp=0xd2e39ff7e0 pc=0x92c865
runtime.systemstack()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:492 +0x4e fp=0xd2e39ff838 sp=0xd2e39ff830 pc=0x973c4e

goroutine 30 [GC worker (idle), 2 minutes]:
runtime.systemstack_switch()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:459 fp=0xc000525f50 sp=0xc000525f48 pc=0x973be0
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1295 +0x209 fp=0xc000525fe0 sp=0xc000525f50 pc=0x92c4e9
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000525fe8 sp=0xc000525fe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 1 [IO wait, 106 minutes]:
runtime.gopark(0xf8?, 0xc0000316b0?, 0x98?, 0x56?, 0xc0000056c8?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc000415108 sp=0xc0004150e8 pc=0x94b496
runtime.netpollblock(0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:526 +0xf7 fp=0xc000415140 sp=0xc000415108 pc=0x941c97
internal/poll.runtime_pollWait(0x2cbf17e2998, 0x72)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:305 +0x89 fp=0xc000415160 sp=0xc000415140 pc=0x970049
internal/poll.(*pollDesc).wait(0x10c2540?, 0xc000415208?, 0x0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc000415188 sp=0xc000415160 pc=0x9eeeb2
internal/poll.execIO(0xc000005698, 0xc000415210)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:175 +0xe5 fp=0xc0004151e0 sp=0xc000415188 pc=0x9f04e5
internal/poll.(*FD).acceptOne(0xc000005680, 0x1a8, {0xc000274000?, 0x9cee5f?, 0x0?}, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:942 +0x6d fp=0xc000415240 sp=0xc0004151e0 pc=0x9f52cd
internal/poll.(*FD).Accept(0xc000005680, 0xc0004153e8)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:976 +0x1d6 fp=0xc0004152f8 sp=0xc000415240 pc=0x9f5636
net.(*netFD).accept(0xc000005680)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/fd_windows.go:139 +0x65 fp=0xc000415408 sp=0xc0004152f8 pc=0xad1045
net.(*TCPListener).accept(0xc000009380)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/tcpsock_posix.go:142 +0x28 fp=0xc000415438 sp=0xc000415408 pc=0xae5b88
net.(*TCPListener).Accept(0xc000009380)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/tcpsock.go:288 +0x3d fp=0xc000415468 sp=0xc000415438 pc=0xae4bfd
net/http.(*onceCloseListener).Accept(0xc000000000?)
        <autogenerated>:1 +0x2a fp=0xc000415480 sp=0xc000415468 pc=0xbf548a
net/http.(*Server).Serve(0xc0001aa000, {0xe10bb8, 0xc000009380})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3070 +0x385 fp=0xc0004155b0 sp=0xc000415480 pc=0xbd04a5
net/http.(*Server).ListenAndServe(0xc0001aa000)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:2999 +0x7d fp=0xc0004155e0 sp=0xc0004155b0 pc=0xbd00dd
net/http.ListenAndServe(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3255
github.com/h2oai/wave.Run({{0xe0c010, 0x5}, {0xe0c2f8, 0xe}, {0xcfba5a, 0x6}, {0xd2ed56, 0x1}, {0xc000028550, 0x40}, ...})
        /home/runner/work/wave/wave/server.go:160 +0x140f fp=0xc0004158f0 sp=0xc0004155e0 pc=0xca910f
main.main()
        /home/runner/work/wave/wave/cmd/wave/main.go:262 +0x1438 fp=0xc000415f80 sp=0xc0004158f0 pc=0xcb07b8
runtime.main()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:250 +0x1fe fp=0xc000415fe0 sp=0xc000415f80 pc=0x94b0fe
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000415fe8 sp=0xc000415fe0 pc=0x975f41

goroutine 2 [force gc (idle), 2 minutes]:
runtime.gopark(0x14bf5b930b17c?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc000057fb0 sp=0xc000057f90 pc=0x94b496
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:369
runtime.forcegchelper()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:302 +0xb1 fp=0xc000057fe0 sp=0xc000057fb0 pc=0x94b331
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000057fe8 sp=0xc000057fe0 pc=0x975f41
created by runtime.init.6
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:290 +0x25

goroutine 3 [GC sweep wait, 2 minutes]:
runtime.gopark(0x106bd01?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc000059f90 sp=0xc000059f70 pc=0x94b496
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:369
runtime.bgsweep(0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcsweep.go:297 +0xd7 fp=0xc000059fc8 sp=0xc000059f90 pc=0x935677
runtime.gcenable.func1()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:178 +0x26 fp=0xc000059fe0 sp=0xc000059fc8 pc=0x92a1c6
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000059fe8 sp=0xc000059fe0 pc=0x975f41
created by runtime.gcenable
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:178 +0x6b

goroutine 4 [GC scavenge wait, 2 minutes]:
runtime.gopark(0x14be7c04cc134?, 0x1e67fc?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc000069f70 sp=0xc000069f50 pc=0x94b496
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:369
runtime.(*scavengerState).park(0x106c2e0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcscavenge.go:389 +0x53 fp=0xc000069fa0 sp=0xc000069f70 pc=0x9336b3
runtime.bgscavenge(0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgcscavenge.go:622 +0x65 fp=0xc000069fc8 sp=0xc000069fa0 pc=0x933cc5
runtime.gcenable.func2()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:179 +0x26 fp=0xc000069fe0 sp=0xc000069fc8 pc=0x92a166
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000069fe8 sp=0xc000069fe0 pc=0x975f41
created by runtime.gcenable
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:179 +0xaa

goroutine 5 [finalizer wait, 135 minutes]:
runtime.gopark(0x94b837?, 0x102b258?, 0x0?, 0x0?, 0xc00005bf70?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00005be28 sp=0xc00005be08 pc=0x94b496
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:369
runtime.runfinq()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mfinal.go:180 +0x10f fp=0xc00005bfe0 sp=0xc00005be28 pc=0x9292cf
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00005bfe8 sp=0xc00005bfe0 pc=0x975f41
created by runtime.createfing
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mfinal.go:157 +0x45

goroutine 6 [select, 133 minutes]:
runtime.gopark(0xc000419de8?, 0x4?, 0xc8?, 0x9b?, 0xc000419ce8?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00052db70 sp=0xc00052db50 pc=0x94b496
runtime.selectgo(0xc00052dde8, 0xc000419ce0, 0xc0000be930?, 0x0, 0x200?, 0x1)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/select.go:328 +0x7dc fp=0xc00052dcb0 sp=0xc00052db70 pc=0x95983c
github.com/h2oai/wave.(*Broker).run(0xc00014e200)
        /home/runner/work/wave/wave/broker.go:222 +0x138 fp=0xc00052dfc8 sp=0xc00052dcb0 pc=0xc99058
github.com/h2oai/wave.Run.func2()
        /home/runner/work/wave/wave/server.go:91 +0x26 fp=0xc00052dfe0 sp=0xc00052dfc8 pc=0xca9986
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00052dfe8 sp=0xc00052dfe0 pc=0x975f41
created by github.com/h2oai/wave.Run
        /home/runner/work/wave/wave/server.go:91 +0x445

goroutine 18 [IO wait, 135 minutes]:
runtime.gopark(0x0?, 0xc000088018?, 0xc8?, 0x80?, 0xc000088048?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc0000c9750 sp=0xc0000c9730 pc=0x94b496
runtime.netpollblock(0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:526 +0xf7 fp=0xc0000c9788 sp=0xc0000c9750 pc=0x941c97
internal/poll.runtime_pollWait(0x2cbf17e28a8, 0x72)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:305 +0x89 fp=0xc0000c97a8 sp=0xc0000c9788 pc=0x970049
internal/poll.(*pollDesc).wait(0x972918?, 0x9c8fc6?, 0x0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0000c97d0 sp=0xc0000c97a8 pc=0x9eeeb2
internal/poll.execIO(0xc000088018, 0xd97920)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:175 +0xe5 fp=0xc0000c9828 sp=0xc0000c97d0 pc=0x9f04e5
internal/poll.(*FD).Read(0xc000088000, {0xc0001ae000, 0x1000, 0x1000})
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:441 +0x26b fp=0xc0000c98b8 sp=0xc0000c9828 pc=0x9f136b
net.(*netFD).Read(0xc000088000, {0xc0001ae000?, 0x9ef3a6?, 0x0?})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/fd_posix.go:55 +0x29 fp=0xc0000c9900 sp=0xc0000c98b8 pc=0xacef49
net.(*conn).Read(0xc00008e000, {0xc0001ae000?, 0x0?, 0x0?})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/net.go:183 +0x45 fp=0xc0000c9948 sp=0xc0000c9900 pc=0xadea05
net/http.(*connReader).Read(0xc0001a8300, {0xc0001ae000, 0x1000, 0x1000})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:786 +0x171 fp=0xc0000c9998 sp=0xc0000c9948 pc=0xbc53d1
bufio.(*Reader).fill(0xc00006c360)
        /opt/hostedtoolcache/go/1.19.4/x64/src/bufio/bufio.go:106 +0xff fp=0xc0000c99d0 sp=0xc0000c9998 pc=0x99779f
bufio.(*Reader).Peek(0xc00006c360, 0x4)
        /opt/hostedtoolcache/go/1.19.4/x64/src/bufio/bufio.go:144 +0x5d fp=0xc0000c99f0 sp=0xc0000c99d0 pc=0x9978fd
net/http.(*conn).readRequest(0xc000090000, {0xe112b8, 0xc0000372c0})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:991 +0x1f8 fp=0xc0000c9b78 sp=0xc0000c99f0 pc=0xbc6918
net/http.(*conn).serve(0xc000090000, {0xe11360, 0xc0001a8180})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:1916 +0x345 fp=0xc0000c9fb8 sp=0xc0000c9b78 pc=0xbcaea5
net/http.(*Server).Serve.func3()
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3102 +0x2e fp=0xc0000c9fe0 sp=0xc0000c9fb8 pc=0xbd088e
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x975f41
created by net/http.(*Server).Serve
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3102 +0x4db

goroutine 14 [GC worker (idle), 135 minutes]:
runtime.gopark(0x97f4f7?, 0xc0000375d0?, 0x50?, 0xe5?, 0xc00006bfb8?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00006bf50 sp=0xc00006bf30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc00006bfe0 sp=0xc00006bf50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00006bfe8 sp=0xc00006bfe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 44 [GC worker (idle), 2 minutes]:
runtime.gopark(0x14bf5b930b17c?, 0xc00001f180?, 0x18?, 0x14?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc0000cbf50 sp=0xc0000cbf30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc0000cbfe0 sp=0xc0000cbf50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000cbfe8 sp=0xc0000cbfe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 43 [GC worker (idle), 135 minutes]:
runtime.gopark(0x97f4f7?, 0xc0000375d0?, 0xe0?, 0x21?, 0xc0000cdfb8?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc0000cdf50 sp=0xc0000cdf30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc0000cdfe0 sp=0xc0000cdf50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0000cdfe8 sp=0xc0000cdfe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 26 [GC worker (idle), 135 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00051df50 sp=0xc00051df30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc00051dfe0 sp=0xc00051df50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00051dfe8 sp=0xc00051dfe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 27 [GC worker (idle), 133 minutes]:
runtime.gopark(0x144aadc8d63d8?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00051ff50 sp=0xc00051ff30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc00051ffe0 sp=0xc00051ff50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00051ffe8 sp=0xc00051ffe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 28 [GC worker (idle), 133 minutes]:
runtime.gopark(0x144aadf2ce5b4?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc000519f50 sp=0xc000519f30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc000519fe0 sp=0xc000519f50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000519fe8 sp=0xc000519fe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 29 [GC worker (idle), 135 minutes]:
runtime.gopark(0x1448f6c826d24?, 0x0?, 0x0?, 0x0?, 0x0?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc00051bf50 sp=0xc00051bf30 pc=0x94b496
runtime.gcBgMarkWorker()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1235 +0xf1 fp=0xc00051bfe0 sp=0xc00051bf50 pc=0x92c3d1
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc00051bfe8 sp=0xc00051bfe0 pc=0x975f41
created by runtime.gcBgMarkStartWorkers
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/mgc.go:1159 +0x25

goroutine 145 [IO wait, 133 minutes]:
runtime.gopark(0x0?, 0xc000088a18?, 0xc8?, 0x8a?, 0xc000088a48?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/proc.go:363 +0xd6 fp=0xc0003f74d8 sp=0xc0003f74b8 pc=0x94b496
runtime.netpollblock(0x940e7b?, 0xf1821ca8?, 0xcb?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:526 +0xf7 fp=0xc0003f7510 sp=0xc0003f74d8 pc=0x941c97
internal/poll.runtime_pollWait(0x2cbf17e26c8, 0x72)
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/netpoll.go:305 +0x89 fp=0xc0003f7530 sp=0xc0003f7510 pc=0x970049
internal/poll.(*pollDesc).wait(0x55555555555555?, 0x38?, 0x0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0003f7558 sp=0xc0003f7530 pc=0x9eeeb2
internal/poll.execIO(0xc000088a18, 0xd97920)
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:175 +0xe5 fp=0xc0003f75b0 sp=0xc0003f7558 pc=0x9f04e5
internal/poll.(*FD).Read(0xc000088a00, {0xc0000f7000, 0x1000, 0x1000})
        /opt/hostedtoolcache/go/1.19.4/x64/src/internal/poll/fd_windows.go:441 +0x26b fp=0xc0003f7640 sp=0xc0003f75b0 pc=0x9f136b
net.(*netFD).Read(0xc000088a00, {0xc0000f7000?, 0x952f71?, 0xcdee80?})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/fd_posix.go:55 +0x29 fp=0xc0003f7688 sp=0xc0003f7640 pc=0xacef49
net.(*conn).Read(0xc00008e070, {0xc0000f7000?, 0x970c87?, 0xc0002241a0?})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/net.go:183 +0x45 fp=0xc0003f76d0 sp=0xc0003f7688 pc=0xadea05
net/http.(*connReader).Read(0xc0001a89c0, {0xc0000f7000, 0x1000, 0x1000})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:786 +0x171 fp=0xc0003f7720 sp=0xc0003f76d0 pc=0xbc53d1
bufio.(*Reader).fill(0xc000086420)
        /opt/hostedtoolcache/go/1.19.4/x64/src/bufio/bufio.go:106 +0xff fp=0xc0003f7758 sp=0xc0003f7720 pc=0x99779f
bufio.(*Reader).ReadSlice(0xc000086420, 0x88?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/bufio/bufio.go:372 +0x2f fp=0xc0003f77a8 sp=0xc0003f7758 pc=0x99838f
bufio.(*Reader).ReadLine(0xc000086420)
        /opt/hostedtoolcache/go/1.19.4/x64/src/bufio/bufio.go:401 +0x27 fp=0xc0003f77f8 sp=0xc0003f77a8 pc=0x9985e7
net/textproto.(*Reader).readLineSlice(0xc0004882d0)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/textproto/reader.go:56 +0x99 fp=0xc0003f78e8 sp=0xc0003f77f8 pc=0xb504f9
net/textproto.(*Reader).ReadLine(...)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/textproto/reader.go:37
net/http.readRequest(0xc00008e070?)
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/request.go:1036 +0x79 fp=0xc0003f79f0 sp=0xc0003f78e8 pc=0xbc0759
net/http.(*conn).readRequest(0xc000090500, {0xe112b8, 0xc0004265c0})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:994 +0x24a fp=0xc0003f7b78 sp=0xc0003f79f0 pc=0xbc696a
net/http.(*conn).serve(0xc000090500, {0xe11360, 0xc0001a8180})
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:1916 +0x345 fp=0xc0003f7fb8 sp=0xc0003f7b78 pc=0xbcaea5
net/http.(*Server).Serve.func3()
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3102 +0x2e fp=0xc0003f7fe0 sp=0xc0003f7fb8 pc=0xbd088e
runtime.goexit()
        /opt/hostedtoolcache/go/1.19.4/x64/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0003f7fe8 sp=0xc0003f7fe0 pc=0x975f41
created by net/http.(*Server).Serve
        /opt/hostedtoolcache/go/1.19.4/x64/src/net/http/server.go:3102 +0x4db

I have no choice but to restart the server which is not a very practical approach. Can you help me understand how to resolve this?

[mturoci]

Please add more info. When does this happen, what Wave version you are using, how many users are using your app, what is the app doing etc. etc.