Merge branch 'sambanova:main' into jorge/post_call_analysis

fine_tuning_embeddings/fine_tune_embed_model.py

@@ -1,35 +1,33 @@
 import argparse
 import csv
 import glob
-import json
 import logging
 import os
+import pickle  # Import for saving and loading .pkl files
+import sys
 from pathlib import Path
-import pandas as pd
 from random import shuffle
-from tqdm.auto import tqdm
-from typing import List, Tuple
+from typing import List, Union
+from typing import Tuple
+
+import pandas as pd
+from dotenv import load_dotenv
+from llama_index import SimpleDirectoryReader, ServiceContext, VectorStoreIndex
+from llama_index.embeddings import OpenAIEmbedding
 from llama_index.finetuning import (
     generate_qa_embedding_pairs,
     EmbeddingQAFinetuneDataset,
     SentenceTransformersFinetuneEngine,
 )
-from llama_index.embeddings import OpenAIEmbedding
-from sentence_transformers.evaluation import InformationRetrievalEvaluator
-from sentence_transformers import SentenceTransformer
-from llama_index import SimpleDirectoryReader, ServiceContext, VectorStoreIndex
+from llama_index.llms import LangChainLLM
 from llama_index.node_parser import SentenceSplitter
-from llama_index.schema import MetadataMode, TextNode
-from llama_index.llms import LangChainLLM, OpenAI
-from typing import List, Union
-import pickle  # Import for saving and loading .pkl files
-import os
-from pathlib import Path
-from dotenv import load_dotenv
-import sys
+from llama_index.schema import TextNode
+from sentence_transformers import SentenceTransformer
+from sentence_transformers.evaluation import InformationRetrievalEvaluator
+from tqdm.auto import tqdm
 
 sys.path.append("..")
-from src.models.sambanova_endpoint import SambaNovaEndpoint
+from utils.sambanova_endpoint import SambaNovaEndpoint
 
 
 # Setup logging

fine_tuning_embeddings/fine_tuning_embed.ipynb

@@ -26,8 +26,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
+   "execution_count": 1,
+   "metadata": {
+    "is_executing": true
+   },
    "outputs": [],
    "source": [
     "import sys\n",
@@ -37,11 +39,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from src.generate_fine_tune_embed_dataset import CorpusLoader, QueryGenerator, SambaNovaEndpoint, LangChainLLM, OpenAI,save_dict_safely\n",
+    "from fine_tuning_embeddings.src.generate_fine_tune_embed_dataset import CorpusLoader, QueryGenerator, LangChainLLM, OpenAI,save_dict_safely\n",
+    "from utils.sambanova_endpoint import SambaNovaEndpoint\n",
     "import random"
    ]
   },
@@ -236,7 +239,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lma-kg",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -250,9 +253,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

fine_tuning_embeddings/src/generate_fine_tune_embed_dataset.py

@@ -1,22 +1,22 @@
-
-import sys
 import argparse
 import glob
 import json
+import logging
 import os
-import pickle
 import random
 import re
+import sys
 import uuid
-import logging
 from typing import Tuple, Dict, List
-from tqdm import tqdm
 
 from llama_index import SimpleDirectoryReader
+from llama_index.llms import LangChainLLM
 from llama_index.node_parser import SimpleNodeParser
 from llama_index.schema import MetadataMode
-from llama_index.llms import LangChainLLM, OpenAI
-from .sambanova_endpoint import SambaNovaEndpoint
+from tqdm import tqdm
+
+sys.path.append("..")
+from utils.sambanova_endpoint import SambaNovaEndpoint
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

fine_tuning_embeddings/use_sns_embeddings.py

@@ -1,25 +1,18 @@
-import requests
 import numpy as np
 import os
 from sklearn.metrics.pairwise import cosine_similarity
 from dotenv import load_dotenv
 from langchain_core.embeddings import Embeddings
-from langchain_core.pydantic_v1 import BaseModel, Extra, root_validator
 import sys
-import re
-import sys
-import glob
-from langchain.chains import RetrievalQA
 from langchain.prompts import PromptTemplate, ChatPromptTemplate
 from langchain_community.document_loaders import WebBaseLoader
-from langchain_community.vectorstores import FAISS, Chroma
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.chains.combine_documents.stuff import create_stuff_documents_chain
 from langchain.chains import create_retrieval_chain
 
 
 # Use embeddings As Part of Langchain
-from langchain.vectorstores import FAISS, Chroma
+from langchain.vectorstores import Chroma
 
 sys.path.append("..")
 from src.models.sambanova_endpoint import SambaNovaEndpoint

fine_tuning_sql/notebooks/3_inference_sql_model.ipynb

@@ -50,8 +50,9 @@
    "outputs": [],
    "source": [
     "import sys\n",
-    "sys.path.append('../../src/')\n",
-    "from models.sambanova_endpoint import SambaNovaEndpoint\n",
+    "sys.path.append('../')\n",
+    "sys.path.append('../../')\n",
+    "from utils.sambanova_endpoint import SambaNovaEndpoint\n",
     "sql_llm = SambaNovaEndpoint(\n",
     "    base_url=base_url,\n",
     "    project_id=project_id,\n",