feat(agentic): emit rate limit backoff events
Co-Authored-By: Virgil <virgil@lethean.io>
This commit is contained in:
parent
332a464cf6
commit
daf4611bfc
2 changed files with 23 additions and 2 deletions
|
|
@ -235,8 +235,16 @@ func (s *PrepSubsystem) trackFailureRate(agent, status string, startedAt time.Ti
|
|||
if elapsed < 60*time.Second {
|
||||
s.failCount[pool]++
|
||||
if s.failCount[pool] >= 3 {
|
||||
s.backoff[pool] = time.Now().Add(30 * time.Minute)
|
||||
backoffDuration := 30 * time.Minute
|
||||
until := time.Now().Add(backoffDuration)
|
||||
s.backoff[pool] = until
|
||||
s.persistRuntimeState()
|
||||
if s.ServiceRuntime != nil {
|
||||
s.Core().ACTION(messages.RateLimitDetected{
|
||||
Pool: pool,
|
||||
Duration: backoffDuration.String(),
|
||||
})
|
||||
}
|
||||
core.Print(nil, "rate-limit detected for %s — pausing pool for 30 minutes", pool)
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"dappco.re/go/agent/pkg/messages"
|
||||
core "dappco.re/go/core"
|
||||
"dappco.re/go/core/forge"
|
||||
"dappco.re/go/core/process"
|
||||
|
|
@ -111,12 +112,24 @@ func TestDispatch_TrackFailureRate_Good(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestDispatch_TrackFailureRate_Bad(t *testing.T) {
|
||||
s := &PrepSubsystem{ServiceRuntime: core.NewServiceRuntime(testCore, AgentOptions{}), backoff: make(map[string]time.Time), failCount: map[string]int{"codex": 2}}
|
||||
c := core.New()
|
||||
var captured []messages.RateLimitDetected
|
||||
c.RegisterAction(func(_ *core.Core, msg core.Message) core.Result {
|
||||
if ev, ok := msg.(messages.RateLimitDetected); ok {
|
||||
captured = append(captured, ev)
|
||||
}
|
||||
return core.Result{OK: true}
|
||||
})
|
||||
|
||||
s := &PrepSubsystem{ServiceRuntime: core.NewServiceRuntime(c, AgentOptions{}), backoff: make(map[string]time.Time), failCount: map[string]int{"codex": 2}}
|
||||
|
||||
// 3rd fast failure triggers backoff
|
||||
triggered := s.trackFailureRate("codex", "failed", time.Now().Add(-10*time.Second))
|
||||
assert.True(t, triggered)
|
||||
assert.True(t, time.Now().Before(s.backoff["codex"]))
|
||||
require.Len(t, captured, 1)
|
||||
assert.Equal(t, "codex", captured[0].Pool)
|
||||
assert.Equal(t, "30m0s", captured[0].Duration)
|
||||
}
|
||||
|
||||
func TestDispatch_TrackFailureRate_Ugly(t *testing.T) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue