Skip to content

Commit

Permalink
llm: looser checks for minimum memory (ollama#5677)
Browse files Browse the repository at this point in the history
  • Loading branch information
jmorganca authored Jul 13, 2024
1 parent 02fea42 commit ef98803
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion llm/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
// On linux, over-allocating CPU memory will almost always result in an error
if runtime.GOOS == "linux" {
systemMemoryRequired := estimate.TotalSize - estimate.VRAMSize
available := min(systemTotalMemory, systemFreeMemory+systemSwapFreeMemory)
available := systemFreeMemory + systemSwapFreeMemory
if systemMemoryRequired > available {
slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", available, "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "swap", format.HumanBytes2(systemSwapFreeMemory))
return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(systemMemoryRequired), format.HumanBytes2(available))
Expand Down

0 comments on commit ef98803

Please sign in to comment.