Add multiple CPU variants for Intel Mac

This also refines the build process for the ext_server build.

.github/workflows/test.yaml

@@ -86,6 +86,9 @@ jobs:
           - os: windows-latest
             arch: arm64
     runs-on: ${{ matrix.os }}
+    env:
+      GOARCH: ${{ matrix.arch }}
+      CGO_ENABLED: "1"
     steps:
       - uses: actions/checkout@v4
         with:

Dockerfile.build

@@ -10,6 +10,7 @@ COPY llm llm
 
 FROM --platform=linux/amd64 nvidia/cuda:$CUDA_VERSION-devel-centos7 AS cuda-build-amd64
 ARG CMAKE_VERSION
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
@@ -19,6 +20,7 @@ RUN OLLAMA_SKIP_CPU_GENERATE=1 sh gen_linux.sh
 
 FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION-devel-rockylinux8 AS cuda-build-arm64
 ARG CMAKE_VERSION
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/gcc-toolset-10/root/usr/bin:$PATH
@@ -28,6 +30,7 @@ RUN OLLAMA_SKIP_CPU_GENERATE=1 sh gen_linux.sh
 
 FROM --platform=linux/amd64 rocm/dev-centos-7:5.7.1-complete AS rocm-5-build-amd64
 ARG CMAKE_VERSION
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
@@ -38,6 +41,7 @@ RUN OLLAMA_SKIP_CPU_GENERATE=1 sh gen_linux.sh
 
 FROM --platform=linux/amd64 rocm/dev-centos-7:6.0-complete AS rocm-6-build-amd64
 ARG CMAKE_VERSION
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
@@ -50,6 +54,7 @@ FROM --platform=linux/amd64 centos:7 AS cpu-build-amd64
 ARG CMAKE_VERSION
 ARG GOLANG_VERSION
 ARG OLLAMA_CUSTOM_CPU_DEFS
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
@@ -61,6 +66,7 @@ FROM --platform=linux/arm64 centos:7 AS cpu-build-arm64
 ARG CMAKE_VERSION
 ARG GOLANG_VERSION
 ARG OLLAMA_CUSTOM_CPU_DEFS
+ARG CGO_CFLAGS
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /rh_linux_deps.sh
 ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
@@ -72,7 +78,7 @@ RUN sh gen_linux.sh
 FROM --platform=linux/amd64 cpu-build-amd64 AS build-amd64
 ENV CGO_ENABLED 1
 ARG GOFLAGS
-ARG CGO_FLAGS
+ARG CGO_CFLAGS
 WORKDIR /go/src/github.com/jmorganca/ollama
 COPY . .
 COPY --from=cuda-build-amd64 /go/src/github.com/jmorganca/ollama/llm/llama.cpp/build/linux/ llm/llama.cpp/build/linux/
@@ -84,7 +90,7 @@ FROM --platform=linux/arm64 cpu-build-arm64 AS build-arm64
 ENV CGO_ENABLED 1
 ARG GOLANG_VERSION
 ARG GOFLAGS
-ARG CGO_FLAGS
+ARG CGO_CFLAGS
 WORKDIR /go/src/github.com/jmorganca/ollama
 COPY . .
 COPY --from=cuda-build-arm64 /go/src/github.com/jmorganca/ollama/llm/llama.cpp/build/linux/ llm/llama.cpp/build/linux/

llm/dyn_ext_server.c

@@ -5,7 +5,7 @@
 
 #ifdef __linux__
 #include <dlfcn.h>
-#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags | RTLD_DEEPBIND)
+#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
 #define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
 #define LOAD_ERR() strdup(dlerror())
 #define UNLOAD_LIBRARY(handle) dlclose(handle)
@@ -58,8 +58,8 @@ void dyn_init(const char *libPath, struct dynamic_llama_server *s,
       {"", NULL},
   };
 
-  printf("loading %s library\n", libPath);
-  s->handle = LOAD_LIBRARY(libPath, RTLD_NOW);
+  printf("loading library %s\n", libPath);
+  s->handle = LOAD_LIBRARY(libPath, RTLD_GLOBAL|RTLD_NOW);
   if (!s->handle) {
     err->id = -1;
     char *msg = LOAD_ERR();

llm/dyn_ext_server.go

@@ -372,15 +372,6 @@ func updatePath(dir string) {
 		newPath := strings.Join(append([]string{dir}, pathComponents...), ";")
 		log.Printf("Updating PATH to %s", newPath)
 		os.Setenv("PATH", newPath)
-	} else {
-		pathComponents := strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":")
-		for _, comp := range pathComponents {
-			if comp == dir {
-				return
-			}
-		}
-		newPath := strings.Join(append([]string{dir}, pathComponents...), ":")
-		log.Printf("Updating LD_LIBRARY_PATH to %s", newPath)
-		os.Setenv("LD_LIBRARY_PATH", newPath)
 	}
+	// linux and darwin rely on rpath
 }

llm/ext_server/CMakeLists.txt

@@ -2,28 +2,24 @@
 
 set(TARGET ext_server)
 option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
-add_library(${TARGET} STATIC ../../../ext_server/ext_server.cpp)
+if (WIN32)
+    add_library(${TARGET} SHARED ../../../ext_server/ext_server.cpp ../../llama.cpp)
+else()
+    add_library(${TARGET} STATIC ../../../ext_server/ext_server.cpp ../../llama.cpp)
+endif()
 target_include_directories(${TARGET} PRIVATE ../../common)
 target_include_directories(${TARGET} PRIVATE ../..)
 target_include_directories(${TARGET} PRIVATE ../../..)
 target_compile_features(${TARGET} PRIVATE cxx_std_11)
 target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1)
-target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
-target_compile_definitions(${TARGET} PRIVATE
-    SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>
-)
-
-if (BUILD_SHARED_LIBS)
-    set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON)
-    target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD)
-    add_library(ext_server_shared SHARED $<TARGET_OBJECTS:ext_server>)
-    target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT})
-    install(TARGETS ext_server_shared LIBRARY)
-endif()
+target_link_libraries(${TARGET} PRIVATE ggml llava common )
+set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+target_compile_definitions(${TARGET} PRIVATE SERVER_VERBOSE=$<BOOL:${LLAMA_SERVER_VERBOSE}>)
+install(TARGETS ext_server LIBRARY)
 
 if (CUDAToolkit_FOUND)
     target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
     if (WIN32)
-        target_link_libraries(ext_server_shared PRIVATE nvml)
+        target_link_libraries(${TARGET} PRIVATE nvml)
     endif()
 endif()

llm/generate/gen_common.sh

@@ -1,15 +1,44 @@
 # common logic accross linux and darwin
 
 init_vars() {
+    case "${GOARCH}" in
+    "amd64")
+        ARCH="x86_64"
+        ;;
+    "arm64")
+        ARCH="arm64"
+        ;;
+    *)
+        ARCH=$(uname -m | sed -e "s/aarch64/arm64/g")
+    esac
+
     LLAMACPP_DIR=../llama.cpp
     CMAKE_DEFS=""
-    CMAKE_TARGETS="--target ggml --target ggml_static --target llama --target build_info --target common --target ext_server --target llava_static"
+    CMAKE_TARGETS="--target ext_server"
     if echo "${CGO_CFLAGS}" | grep -- '-g' >/dev/null; then
-        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_GPROF=on -DLLAMA_SERVER_VERBOSE=on"
+        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_GPROF=on -DLLAMA_SERVER_VERBOSE=on ${CMAKE_DEFS}"
     else
         # TODO - add additional optimization flags...
-        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=Release -DLLAMA_SERVER_VERBOSE=off"
+        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=Release -DLLAMA_SERVER_VERBOSE=off ${CMAKE_DEFS}"
     fi
+    case $(uname -s) in 
+    "Darwin")
+        LIB_EXT="dylib"
+        WHOLE_ARCHIVE="-Wl,-force_load"
+        NO_WHOLE_ARCHIVE=""
+        GCC_ARCH="-arch ${ARCH}"
+        ;;
+    "Linux")
+        LIB_EXT="so"
+        WHOLE_ARCHIVE="-Wl,--whole-archive"
+        NO_WHOLE_ARCHIVE="-Wl,--no-whole-archive"
+
+        # Cross compiling not supported on linux - Use docker
+        GCC_ARCH=""
+        ;;
+    *)
+        ;;
+    esac
 }
 
 git_module_setup() {
@@ -40,25 +69,29 @@ apply_patches() {
 build() {
     cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS}
     cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8
+    mkdir -p ${BUILD_DIR}/lib/
+    g++ -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.${LIB_EXT} \
+        ${GCC_ARCH} \
+        ${WHOLE_ARCHIVE} ${BUILD_DIR}/examples/server/libext_server.a ${NO_WHOLE_ARCHIVE} \
+        ${BUILD_DIR}/common/libcommon.a \
+        ${BUILD_DIR}/libllama.a \
+        -Wl,-rpath,\$ORIGIN \
+        -lpthread -ldl -lm \
+        ${EXTRA_LIBS}
 }
 
-install() {
-    rm -rf ${BUILD_DIR}/lib
-    mkdir -p ${BUILD_DIR}/lib
-    cp ${BUILD_DIR}/examples/server/libext_server.a ${BUILD_DIR}/lib
-    cp ${BUILD_DIR}/common/libcommon.a ${BUILD_DIR}/lib
-    cp ${BUILD_DIR}/libllama.a ${BUILD_DIR}/lib
-    cp ${BUILD_DIR}/libggml_static.a ${BUILD_DIR}/lib
-}
-
-link_server_lib() {
-    gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
-        -Wl,--whole-archive \
-        ${BUILD_DIR}/lib/libext_server.a \
-        -Wl,--no-whole-archive \
-        ${BUILD_DIR}/lib/libcommon.a \
-        ${BUILD_DIR}/lib/libllama.a \
-        -lstdc++
+compress_libs() {
+    echo "Compressing payloads to reduce overall binary size..."
+    pids=""
+    for lib in ${BUILD_DIR}/lib/*.${LIB_EXT}* ; do
+        bzip2 -v9 ${lib} &
+        pids+=" $!"
+    done
+    echo 
+    for pid in ${pids}; do
+        wait $pid
+    done
+    echo "Finished compression"
 }
 
 # Keep the local tree clean after we're done with the build

llm/generate/gen_darwin.sh

@@ -9,16 +9,52 @@ set -o pipefail
 echo "Starting darwin generate script"
 source $(dirname $0)/gen_common.sh
 init_vars
-CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DCMAKE_SYSTEM_NAME=Darwin -DLLAMA_ACCELERATE=on ${CMAKE_DEFS}"
-BUILD_DIR="${LLAMACPP_DIR}/build/darwin/metal"
+git_module_setup
+apply_patches
+
+COMMON_DARWIN_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DCMAKE_SYSTEM_NAME=Darwin -DLLAMA_ACCELERATE=off"
+
 case "${GOARCH}" in
 "amd64")
-    CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLAMA_METAL=off -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
-    ARCH="x86_64"
+    COMMON_CPU_DEFS="${COMMON_DARWIN_DEFS} -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DLLAMA_METAL=off -DLLAMA_NATIVE=off"
+
+    #
+    # CPU first for the default library, set up as lowest common denominator for maximum compatibility (including Rosetta)
+    #
+    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+    BUILD_DIR="${LLAMACPP_DIR}/build/darwin/${ARCH}/cpu"
+    echo "Building LCD CPU"
+    build
+    compress_libs
+
+    #
+    # ~2011 CPU Dynamic library with more capabilities turned on to optimize performance
+    # Approximately 400% faster than LCD on same CPU
+    #
+    init_vars
+    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+    BUILD_DIR="${LLAMACPP_DIR}/build/darwin/${ARCH}/cpu_avx"
+    echo "Building AVX CPU"
+    build
+    compress_libs
+
+    #
+    # ~2013 CPU Dynamic library
+    # Approximately 10% faster than AVX on same CPU
+    #
+    init_vars
+    CMAKE_DEFS="${COMMON_CPU_DEFS} -DLLAMA_AVX=on -DLLAMA_AVX2=on -DLLAMA_AVX512=off -DLLAMA_FMA=on -DLLAMA_F16C=on ${CMAKE_DEFS}"
+    BUILD_DIR="${LLAMACPP_DIR}/build/darwin/${ARCH}/cpu_avx2"
+    echo "Building AVX2 CPU"
+    build
+    compress_libs
     ;;
 "arm64")
-    CMAKE_DEFS="-DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DLLAMA_METAL=on ${CMAKE_DEFS}"
-    ARCH="arm64"
+    CMAKE_DEFS="${COMMON_DARWIN_DEFS} -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DCMAKE_OSX_ARCHITECTURES=${ARCH} -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on ${CMAKE_DEFS}"
+    BUILD_DIR="${LLAMACPP_DIR}/build/darwin/${ARCH}/metal"
+    EXTRA_LIBS="${EXTRA_LIBS} -framework Accelerate -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders"
+    build
+    compress_libs
     ;;
 *)
     echo "GOARCH must be set"
@@ -27,21 +63,4 @@ case "${GOARCH}" in
     ;;
 esac
 
-git_module_setup
-apply_patches
-build
-install
-gcc -fPIC -g -shared -o ${BUILD_DIR}/lib/libext_server.so \
-    -arch ${ARCH} \
-    -Wl,-force_load ${BUILD_DIR}/lib/libext_server.a \
-    ${BUILD_DIR}/lib/libcommon.a \
-    ${BUILD_DIR}/lib/libllama.a \
-    ${BUILD_DIR}/lib/libggml_static.a \
-    -lpthread -ldl -lm -lc++ \
-    -framework Accelerate \
-    -framework Foundation \
-    -framework Metal \
-    -framework MetalKit \
-    -framework MetalPerformanceShaders
-
 cleanup