修改版本号

.github/workflows/release.yml

@@ -71,7 +71,7 @@ jobs:
         run: |
           cd ./src/backend
           poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
-          sed -i 's/^bisheng_langchain.*/bisheng_langchain = "0.0.0"/g' pyproject.toml
+          # sed -i 's/^bisheng_langchain.*/bisheng_langchain = "0.0.0"/g' pyproject.toml
           poetry lock
           cd ../../
 

docker/bisheng/config/config.yaml

@@ -43,7 +43,11 @@ agents:
     documentation: ""
   SQLAgent:
     documentation: ""
+  ChatglmFunctionsAgent:
+    documentation: ""
 chains:
+  RetrievalChain:
+    documentation: ""
   RuleBasedRouter:
     documentation: ""
   MultiRuleChain:
@@ -189,11 +193,13 @@ llms:
     documentation: ""
   ChatWenxin:
     documentation: ""
+  ChatQWen:
+    documentation: ""
   ChatZhipuAI:
     documentation: ""
   ChatXunfeiAI:
     documentation: ""
-  HostChatGLM2:
+  HostChatGLM:
     documentation: ""
   HostBaichuanChat:
     documentation: ""

src/backend/bisheng/api/v1/knowledge.py

@@ -600,7 +600,7 @@ def text_knowledge(
         metadata = [{
             'file_id': db_file.id,
             'knowledge_id': f'{db_knowledge.id}',
-            'page': doc.metadata.pop('page', ''),
+            'page': doc.metadata.pop('page'),
             'source': doc.metadata.get('source', ''),
             'bbox': doc.metadata.get('bbox', ''),
             'extra': json.dumps(doc.metadata)

src/backend/bisheng/chat/manager.py

@@ -235,7 +235,7 @@ async def handle_websocket(
                             return
         except Exception as e:
             # Handle any exceptions that might occur
-            logger.exception(e)
+            logger.error()
             await self.close_connection(
                 client_id=client_id,
                 chat_id=chat_id,

src/backend/bisheng/interface/embeddings/custom.py

@@ -1,33 +1,28 @@
-import json
 from typing import List
 
-from bisheng.utils.logger import logger
-from bisheng_langchain.utils.requests import TextRequestsWrapper
+from bisheng.settings import settings
 from langchain.embeddings.base import Embeddings
+from langchain.embeddings.openai import OpenAIEmbeddings
 
 
 class OpenAIProxyEmbedding(Embeddings):
-    request = TextRequestsWrapper()
+
+    def __init__(self) -> None:
+        param = settings.get_knowledge().get('embeddings').get('text-embedding-ada-002')
+        self.embd = OpenAIEmbeddings(**param)
+        super().__init__()
 
     @classmethod
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
         if not texts:
             return []
         """Embed search docs."""
-        texts = [text for text in texts if text]
-        data = {'texts': texts}
-
-        resp = self.request.post(url='http://43.133.35.137:8080/chunks_embed', json=data)
-        logger.info(f'texts={texts}')
-        return json.loads(resp).get('data')
+        return self.embd.embed_documents(texts)
 
     @classmethod
     def embed_query(self, text: str) -> List[float]:
         """Embed query text."""
-        data = {'query': [text]}
-        resp = self.request.post(url='http://43.133.35.137:8080/query_embed', json=data)
-        logger.info(f'texts={data}')
-        return json.loads(resp).get('data')[0]
+        return self.embed_query(text)
 
 
 CUSTOM_EMBEDDING = {

src/backend/bisheng/interface/initialize/loading.py

@@ -456,7 +456,7 @@ def instantiate_vectorstore(class_object: Type[VectorStore], params: Dict):
     # ! This might not work. Need to test
     if search_kwargs and hasattr(vecstore, 'as_retriever'):
         if settings.get_from_db('file_access'):
-            # need to verify file access
+            # need to verify file access / 只针对知识库
             access_url = settings.get_from_db('file_access') + f'?username={user_name}'
             vecstore = VectorStoreFilterRetriever(vectorstore=vecstore,
                                                   search_type=search_type,

src/backend/pyproject.toml

@@ -18,7 +18,7 @@ include = ["./bisheng/*", "bisheng/**/*"]
 bisheng = "bisheng.__main__:main"
 
 [tool.poetry.dependencies]
-bisheng_langchain = "0.2.0rc1"
+bisheng_langchain = "0.2.1.1"
 bisheng_pyautogen = "0.1.18"
 minio = "^7.2.0"
 fastapi_jwt_auth = "^0.5.0"

src/backend/test/milvus_trans.py

@@ -0,0 +1,27 @@
+from bisheng_langchain.embeddings import HostEmbeddings
+from bisheng_langchain.vectorstores import Milvus
+from sqlmodel import Session, create_engine
+
+params = {}
+params['connection_args'] = {
+    'host': '192.168.106.116',
+    'port': '19530',
+    'user': '',
+    'password': '',
+    'secure': False
+}
+params['documents'] = []
+embedding = HostEmbeddings(model='multilingual-e5-large',
+                           host_base_url='http://192.168.106.12:9001/v2.1/models')
+
+database_url = 'mysql+pymysql://root:[email protected]:3306/langflow?charset=utf8mb4'
+engine = create_engine(database_url, connect_args={}, pool_pre_ping=True)
+with Session(engine) as session:
+    db_knowledge = session.exec('select id, collection_name, model from knowledge').all()
+    for knowledge in db_knowledge:
+
+        if knowledge[1].startswith('col'):
+            params['collection_name'] = knowledge[1]
+            cli = Milvus.from_documents(embedding=embedding, **params)
+            li = cli.col.query(expr='file_id>1')
+            pass

src/backend/test/test_api.py

@@ -25,6 +25,7 @@ def _test_python_code():
 
 
 def _test_uns():
+
     import base64
     import requests
 

src/bisheng-langchain/bisheng_langchain/vectorstores/retriever.py

@@ -78,14 +78,14 @@ async def _aget_relevant_documents(
         return self.get_file_access(docs)
 
     def get_file_access(self, docs: List[Document]):
-        file_ids = [doc.metadata.get('file_id') for doc in docs]
+        file_ids = [doc.metadata.get('file_id') for doc in docs if 'file_id' in doc.metadata]
         if file_ids:
             res = requests.get(self.access_url, json=file_ids)
             if res.status_code == 200:
                 doc_res = res.json().get('data') or []
                 doc_right = {doc.get('docid') for doc in doc_res if doc.get('result') == 1}
                 for doc in docs:
-                    if doc.metadata.get('file_id') not in doc_right:
+                    if doc.metadata.get('file_id') and doc.metadata.get('file_id') not in doc_right:
                         doc.page_content = ''
                         doc.metadata['right'] = False
                 return docs

src/bisheng-langchain/version.txt

@@ -1 +1 @@
-v0.1.10
+v0.2.1.1