Skip to content

Commit

Permalink
logs: explain client aborts better (ollama#7783)
Browse files Browse the repository at this point in the history
Users get confused by "Failed to acquire semaphore" error="context canceled"
messages in the logs, which are actually clients giving up.  While there could be
a legitimate hang bug in the system, sometimes this is just short client timeouts
with an overloaded system, so this should help users understand what's going on
better.
  • Loading branch information
dhiltgen authored Nov 22, 2024
1 parent d88972e commit b85520b
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 4 deletions.
12 changes: 10 additions & 2 deletions llama/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,11 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {

// Ensure that a place to put the sequence is available
if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
slog.Error("Failed to acquire semaphore", "error", err)
if errors.Is(err, context.Canceled) {
slog.Info("aborting completion request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err)
}
return
}
defer s.seqsSem.Release(1)
Expand Down Expand Up @@ -740,7 +744,11 @@ func (s *Server) embeddings(w http.ResponseWriter, r *http.Request) {

// Ensure that a place to put the sequence is available
if err := s.seqsSem.Acquire(r.Context(), 1); err != nil {
slog.Error("Failed to acquire semaphore", "error", err)
if errors.Is(err, context.Canceled) {
slog.Info("aborting embeddings request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err)
}
return
}
defer s.seqsSem.Release(1)
Expand Down
12 changes: 10 additions & 2 deletions llm/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,11 @@ type CompletionResponse struct {

func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error {
if err := s.sem.Acquire(ctx, 1); err != nil {
slog.Error("Failed to acquire semaphore", "error", err)
if errors.Is(err, context.Canceled) {
slog.Info("aborting completion request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err)
}
return err
}
defer s.sem.Release(1)
Expand Down Expand Up @@ -865,7 +869,11 @@ type EmbeddingResponse struct {

func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) {
if err := s.sem.Acquire(ctx, 1); err != nil {
slog.Error("Failed to acquire semaphore", "error", err)
if errors.Is(err, context.Canceled) {
slog.Info("aborting embedding request due to client closing the connection")
} else {
slog.Error("Failed to acquire semaphore", "error", err)
}
return nil, err
}
defer s.sem.Release(1)
Expand Down

0 comments on commit b85520b

Please sign in to comment.