fix: make Restart command return immediately
The Restart() method was calling startInternal() which waits for the gateway to exit. This is wrong for manual restart commands. Fixed by: - Creating startLocked() that starts and returns immediately - Having Restart() call startLocked() for immediate return - Adding RunWithAutoRestart() for crash monitoring with retry Now manual 'picoclaw gateway restart' works as expected: stops the gateway, starts it, and returns immediately.
This commit is contained in:
parent
1b75778658
commit
072c4f9a39
1 changed files with 97 additions and 53 deletions
|
|
@ -241,18 +241,13 @@ func (s *Service) stopLocked() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Restart restarts the gateway daemon with automatic crash recovery.
|
// Restart restarts the gateway daemon (stop if running, then start).
|
||||||
//
|
// This is a simple one-time restart. For auto-restart with crash recovery,
|
||||||
// This method implements the auto-restart loop:
|
// use the Run method with proper supervision.
|
||||||
// 1. If daemon is running, stop it first
|
|
||||||
// 2. Start the daemon
|
|
||||||
// 3. If it crashes, wait (exponential backoff) and restart
|
|
||||||
// 4. Repeat up to MaxAttempts within WindowDuration
|
|
||||||
// 5. Give up if max attempts exceeded
|
|
||||||
//
|
//
|
||||||
// Returns an error if:
|
// Returns an error if:
|
||||||
// - Stop fails
|
// - Stop fails
|
||||||
// - Max restart attempts exceeded
|
// - Start fails
|
||||||
func (s *Service) Restart() error {
|
func (s *Service) Restart() error {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
|
|
@ -266,61 +261,98 @@ func (s *Service) Restart() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start the gateway (non-blocking, returns immediately)
|
||||||
|
if err := s.startLocked(); err != nil {
|
||||||
|
return fmt.Errorf("failed to start daemon: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.InfoC("daemon", "Gateway daemon restarted successfully")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunWithAutoRestart starts the gateway and monitors it for crashes.
|
||||||
|
// If the gateway crashes, it will automatically restart with exponential backoff.
|
||||||
|
// This method blocks until the gateway crashes 3 times within 5 minutes.
|
||||||
|
//
|
||||||
|
// Use this for long-running daemon supervision.
|
||||||
|
func (s *Service) RunWithAutoRestart() error {
|
||||||
// Create restart tracker
|
// Create restart tracker
|
||||||
tracker := NewRestartTracker(s.restartPolicy)
|
tracker := NewRestartTracker(s.restartPolicy)
|
||||||
|
|
||||||
// Restart loop
|
// Restart loop with crash recovery
|
||||||
for tracker.ShouldRestart() {
|
for tracker.ShouldRestart() {
|
||||||
// Attempt to start
|
// Start the gateway and wait for it to exit
|
||||||
err := s.startInternal()
|
err := s.startInternal()
|
||||||
if err == nil {
|
if err != nil {
|
||||||
// Success! Reset restart counter and return
|
// Gateway exited with an error
|
||||||
tracker.Reset()
|
|
||||||
logger.InfoC("daemon", "Gateway daemon restarted successfully")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this is an "already running" error
|
// Check if this is an "already running" error
|
||||||
if _, ok := err.(*AlreadyRunningError); ok {
|
if _, ok := err.(*AlreadyRunningError); ok {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record the failed attempt and get backoff duration
|
// Record the failed attempt and get backoff duration
|
||||||
backoff, backoffErr := tracker.RecordAttempt()
|
backoff, backoffErr := tracker.RecordAttempt()
|
||||||
if backoffErr != nil {
|
if backoffErr != nil {
|
||||||
// Max attempts exceeded
|
// Max attempts exceeded
|
||||||
logger.ErrorCF("daemon", "Maximum restart attempts exceeded", map[string]any{
|
logger.ErrorCF("daemon", "Maximum restart attempts exceeded", map[string]any{
|
||||||
"attempts": tracker.GetAttemptCount(),
|
"attempts": tracker.GetAttemptCount(),
|
||||||
"max": s.restartPolicy.MaxAttempts,
|
"max": s.restartPolicy.MaxAttempts,
|
||||||
|
})
|
||||||
|
return backoffErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update state with restart count
|
||||||
|
s.state.IncrementRestartCount()
|
||||||
|
|
||||||
|
logger.WarnCF("daemon", "Gateway daemon crashed, will restart", map[string]any{
|
||||||
|
"attempt": tracker.GetAttemptCount(),
|
||||||
|
"backoff": backoff.String(),
|
||||||
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
return backoffErr
|
|
||||||
}
|
// Wait before next restart attempt
|
||||||
|
select {
|
||||||
// Update state with restart count
|
case <-time.After(backoff):
|
||||||
s.state.IncrementRestartCount()
|
// Continue to next attempt
|
||||||
|
continue
|
||||||
logger.WarnCF("daemon", "Gateway daemon crashed, will restart", map[string]any{
|
case <-s.quitChan:
|
||||||
"attempt": tracker.GetAttemptCount(),
|
// Abort restart loop
|
||||||
"backoff": backoff.String(),
|
return fmt.Errorf("restart aborted")
|
||||||
"error": err.Error(),
|
}
|
||||||
})
|
|
||||||
|
|
||||||
// Wait before next restart attempt
|
|
||||||
select {
|
|
||||||
case <-time.After(backoff):
|
|
||||||
// Continue to next attempt
|
|
||||||
case <-s.quitChan:
|
|
||||||
// Abort restart loop
|
|
||||||
return fmt.Errorf("restart aborted")
|
|
||||||
}
|
}
|
||||||
|
// If startInternal returned nil, the gateway was stopped manually
|
||||||
|
// Exit the loop and return
|
||||||
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("maximum restart attempts exceeded")
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// startInternal starts the daemon without locking.
|
// startInternal starts the daemon without locking and waits for it to exit.
|
||||||
// Must be called with the lock held.
|
// Must be called with the lock held.
|
||||||
|
// This is used for the auto-restart loop where we monitor for crashes.
|
||||||
func (s *Service) startInternal() error {
|
func (s *Service) startInternal() error {
|
||||||
|
// Start the gateway (non-blocking)
|
||||||
|
if err := s.startLocked(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the process to exit
|
||||||
|
pid := s.pidFile.Read()
|
||||||
|
process, err := os.FindProcess(pid)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to find process: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Block until process exits
|
||||||
|
_, err = process.Wait()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// startLocked starts the gateway daemon without locking and returns immediately.
|
||||||
|
// Must be called with the lock held.
|
||||||
|
func (s *Service) startLocked() error {
|
||||||
// Prepare command
|
// Prepare command
|
||||||
cmd := exec.Command(s.binaryPath, s.buildArgs()...)
|
cmd := exec.Command(s.binaryPath, s.buildArgs()...)
|
||||||
cmd.Env = append(os.Environ(), "PICOCLAW_DAEMON=1")
|
cmd.Env = append(os.Environ(), "PICOCLAW_DAEMON=1")
|
||||||
|
|
@ -342,20 +374,32 @@ func (s *Service) startInternal() error {
|
||||||
|
|
||||||
pid := cmd.Process.Pid
|
pid := cmd.Process.Pid
|
||||||
|
|
||||||
// Write PID file
|
// Write PID file with the child's PID
|
||||||
if err := s.pidFile.Write(); err != nil {
|
if err := s.pidFile.WritePID(pid); err != nil {
|
||||||
cmd.Process.Kill()
|
cmd.Process.Kill()
|
||||||
logFile.Close()
|
logFile.Close()
|
||||||
return fmt.Errorf("failed to write PID file: %w", err)
|
return fmt.Errorf("failed to write PID file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verify the child process is actually running
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if !s.pidFile.IsProcessRunning() {
|
||||||
|
s.pidFile.Remove()
|
||||||
|
logFile.Close()
|
||||||
|
return fmt.Errorf("gateway process failed to start (check log file: %s)", s.logConfig.Path)
|
||||||
|
}
|
||||||
|
|
||||||
// Update state
|
// Update state
|
||||||
s.state.SetPID(pid)
|
s.state.SetPID(pid)
|
||||||
s.state.SetStartTime(time.Now())
|
s.state.SetStartTime(time.Now())
|
||||||
s.state.SetVersion(s.version)
|
s.state.SetVersion(s.version)
|
||||||
|
|
||||||
// Wait for process to exit
|
logger.InfoCF("daemon", "Gateway daemon started", map[string]any{
|
||||||
return cmd.Wait()
|
"pid": pid,
|
||||||
|
"version": s.version,
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status returns the current status of the gateway daemon.
|
// Status returns the current status of the gateway daemon.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue