Update llama.cpp submodule commit to d94c6e0c (ollama#5805)

jay74jung · Jul 22, 2024 · f8fedbd · f8fedbd
1 parent b3e5491
commit f8fedbd
Show file tree

Hide file tree

Showing 9 changed files with 366 additions and 81 deletions.
diff --git a/llm/llama.cpp b/llm/llama.cpp
diff --git a/llm/patches/05-default-pretokenizer.diff b/llm/patches/05-default-pretokenizer.diff
@@ -1,8 +1,8 @@
 diff --git a/src/llama.cpp b/src/llama.cpp
-index 2b9ace28..172640e2 100644
+index 8fe51971..7113ba64 100644
 --- a/src/llama.cpp
 +++ b/src/llama.cpp
-@@ -5357,16 +5357,7 @@ static void llm_load_vocab(
+@@ -5433,16 +5433,7 @@ static void llm_load_vocab(
          if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
              vocab.tokenizer_add_space_prefix = false;
              vocab.tokenizer_clean_spaces = true;
@@ -20,9 +20,9 @@ index 2b9ace28..172640e2 100644
                  vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
              } else if (
                      tokenizer_pre == "llama3"   ||
-@@ -5439,7 +5430,8 @@ static void llm_load_vocab(
-                 tokenizer_pre == "jais") {
-                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_JAIS;
+@@ -5526,7 +5517,8 @@ static void llm_load_vocab(
+                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_SMOLLM;
+                 vocab.tokenizer_clean_spaces = false;
              } else {
 -                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
 +                LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);

diff --git a/llm/patches/07-embeddings.diff → llm/patches/06-embeddings.diff b/llm/patches/07-embeddings.diff → llm/patches/06-embeddings.diff
diff --git a/llm/patches/06-qwen2.diff b/llm/patches/06-qwen2.diff
diff --git a/llm/patches/08-clip-unicode.diff → llm/patches/07-clip-unicode.diff b/llm/patches/08-clip-unicode.diff → llm/patches/07-clip-unicode.diff
diff --git a/llm/patches/09-pooling.diff → llm/patches/08-pooling.diff b/llm/patches/09-pooling.diff → llm/patches/08-pooling.diff