use voyage with truncation (langchain-ai#270)

ublrama · Mar 12, 2024 · 7f32b3d · 7f32b3d
1 parent e17d808
commit 7f32b3d
Show file tree

Hide file tree

Showing 4 changed files with 152 additions and 130 deletions.
diff --git a/.github/workflows/clear-and-update-index.yml b/.github/workflows/clear-and-update-index.yml
@@ -35,5 +35,6 @@ jobs:
           WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }}
           WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
           RECORD_MANAGER_DB_URL: ${{ secrets.RECORD_MANAGER_DB_URL }}
+          VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           VOYAGE_AI_MODEL: ${{ secrets.VOYAGE_AI_MODEL }}
           VOYAGE_AI_URL: ${{ secrets.VOYAGE_AI_URL }}
diff --git a/backend/ingest.py b/backend/ingest.py
@@ -11,6 +11,7 @@
 from langchain.indexes import SQLRecordManager, index
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.utils.html import PREFIXES_TO_IGNORE_REGEX, SUFFIXES_TO_IGNORE_REGEX
+from langchain_community.embeddings import VoyageEmbeddings
 from langchain_community.vectorstores import Weaviate
 from langchain_core.embeddings import Embeddings
 from langchain_openai import OpenAIEmbeddings
@@ -20,6 +21,8 @@
 
 
 def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_API_KEY") and os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings(model=os.environ["VOYAGE_AI_MODEL"], truncation=True)
     return OpenAIEmbeddings(model="text-embedding-3-small", chunk_size=200)