diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go
index 6a4fbd169..3c03dd957 100644
--- a/pkg/agent/loop.go
+++ b/pkg/agent/loop.go
@@ -1463,7 +1463,7 @@ func (al *AgentLoop) retryLLMCall(
 			return resp, nil
 		}
 		if attempt < maxRetries-1 {
-			time.Sleep(time.Duration(attempt+1) * time.Second)
+			time.Sleep(100 * time.Millisecond)
 		}
 	}