forked from sambanova/ai-starter-kit
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Sambaverse and Sambastudio langchain wrapper usage notebook
- Loading branch information
1 parent
4c5193e
commit 68cf6c8
Showing
1 changed file
with
261 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,261 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 1, | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "data": { | ||
| "text/plain": [ | ||
| "True" | ||
| ] | ||
| }, | ||
| "execution_count": 1, | ||
| "metadata": {}, | ||
| "output_type": "execute_result" | ||
| } | ||
| ], | ||
| "source": [ | ||
| "import os\n", | ||
| "import sys\n", | ||
| "from sambanova_endpoint import SambaNovaEndpoint, SambaverseEndpoint\n", | ||
| "from dotenv import load_dotenv\n", | ||
| "import json\n", | ||
| "load_dotenv(\"../.env\")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# SambaStudio endpoint" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Non streaming" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 2, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "\n", | ||
| "llm = SambaNovaEndpoint(\n", | ||
| " streaming=False,\n", | ||
| " model_kwargs={\n", | ||
| " \"do_sample\": True, \n", | ||
| " \"temperature\": 0.01,\n", | ||
| " \"max_tokens_to_generate\": 256,\n", | ||
| " # \"repetition_penalty\": {\"type\": \"float\", \"value\": \"1\"},\n", | ||
| " # \"top_k\": {\"type\": \"int\", \"value\": \"50\"},\n", | ||
| " # \"top_logprobs\": {\"type\": \"int\", \"value\": \"0\"},\n", | ||
| " # \"top_p\": {\"type\": \"float\", \"value\": \"1\"}\n", | ||
| " }\n", | ||
| " ) \n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 3, | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "data": { | ||
| "text/plain": [ | ||
| "\"of a man who was once a great warrior, but is now a humble farmer.\\n\\nThe sun sets on the fields, a weary farmer bends to his work. Once a great warrior, he fought for his people's freedom. Now, he fights for his family's survival. His calloused hands, a testament to his past, still hold the strength of a warrior. But his heart, once filled with anger and vengeance, now holds only love and hope.\"" | ||
| ] | ||
| }, | ||
| "execution_count": 3, | ||
| "metadata": {}, | ||
| "output_type": "execute_result" | ||
| } | ||
| ], | ||
| "source": [ | ||
| "llm.invoke(\"tell me a 50 word tale\")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Streaming" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 4, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "llm=SambaNovaEndpoint(\n", | ||
| " streaming=True,\n", | ||
| " model_kwargs={\n", | ||
| " \"do_sample\": True,\n", | ||
| " \"max_tokens_to_generate\": 256,\n", | ||
| " \"temperature\": 0.01,\n", | ||
| " # \"repetition_penalty\": {\"type\": \"float\", \"value\": \"1\"},\n", | ||
| " # \"top_k\": {\"type\": \"int\", \"value\": \"50\"},\n", | ||
| " # \"top_logprobs\": {\"type\": \"int\", \"value\": \"0\"},\n", | ||
| " # \"top_p\": {\"type\": \"float\", \"value\": \"1\"}\n", | ||
| " }\n", | ||
| " )" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 5, | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "name": "stdout", | ||
| "output_type": "stream", | ||
| "text": [ | ||
| " of a man who was once a great warrior, but is now a humble farmer.\n", | ||
| "\n", | ||
| "The sun sets on the fields, a weary farmer bends to his work. Once a great warrior, he fought for his people's freedom. Now, he fights for his family's survival. His calloused hands, a testament to his past, still hold the strength of a warrior. But his heart, once filled with anger and vengeance, now holds only love and hope." | ||
| ] | ||
| } | ||
| ], | ||
| "source": [ | ||
| "for chunk in llm.stream(\"tell me a 50 word tale\"):\n", | ||
| " data = json.loads(json.loads(chunk)[\"data\"])[\"stream_token\"]\n", | ||
| " print(data, end=\"\", flush=True)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "# Sambaverse\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Non Streaming" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 6, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "llm = SambaverseEndpoint(\n", | ||
| " sambaverse_model_name=\"Meta/llama-2-7b-chat-hf\",\n", | ||
| " model_kwargs={\n", | ||
| " \"do_sample\": True, \n", | ||
| " \"max_tokens_to_generate\": 256,\n", | ||
| " \"temperature\": 0.01,\n", | ||
| " \"process_prompt\": True,\n", | ||
| " \"select_expert\": \"llama-2-7b-chat-hf\"\n", | ||
| " #\"stop_sequences\": { \"type\":\"str\", \"value\":\"\"},\n", | ||
| " # \"repetition_penalty\": {\"type\": \"float\", \"value\": \"1\"},\n", | ||
| " # \"top_k\": {\"type\": \"int\", \"value\": \"50\"},\n", | ||
| " # \"top_p\": {\"type\": \"float\", \"value\": \"1\"}\n", | ||
| " }\n", | ||
| " ) " | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 7, | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "data": { | ||
| "text/plain": [ | ||
| "' Sure! Here is a 50-word tale:\\n\\nThe cat purred contentedly on my lap, pawing at my hand with a gentle mew.'" | ||
| ] | ||
| }, | ||
| "execution_count": 7, | ||
| "metadata": {}, | ||
| "output_type": "execute_result" | ||
| } | ||
| ], | ||
| "source": [ | ||
| "llm.invoke(\"tell me a 50 word tale\")" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "## Streaming" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 8, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "llm = SambaverseEndpoint(\n", | ||
| " streaming=True,\n", | ||
| " sambaverse_model_name=\"Meta/llama-2-7b-chat-hf\",\n", | ||
| " model_kwargs={\n", | ||
| " \"do_sample\": True, \n", | ||
| " \"max_tokens_to_generate\": 256,\n", | ||
| " \"temperature\": 0.01,\n", | ||
| " \"process_prompt\": True,\n", | ||
| " \"select_expert\": \"llama-2-7b-chat-hf\"\n", | ||
| " #\"stop_sequences\": { \"type\":\"str\", \"value\":\"\"},\n", | ||
| " # \"repetition_penalty\": {\"type\": \"float\", \"value\": \"1\"},\n", | ||
| " # \"top_k\": {\"type\": \"int\", \"value\": \"50\"},\n", | ||
| " # \"top_p\": {\"type\": \"float\", \"value\": \"1\"}\n", | ||
| " }\n", | ||
| " ) " | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": 9, | ||
| "metadata": {}, | ||
| "outputs": [ | ||
| { | ||
| "name": "stdout", | ||
| "output_type": "stream", | ||
| "text": [ | ||
| " Sure! Here is a 50-word tale:\n", | ||
| "\n", | ||
| "The cat purred contentedly on my lap, pawing at my hand with a gentle mew." | ||
| ] | ||
| } | ||
| ], | ||
| "source": [ | ||
| "\n", | ||
| "for chunk in llm.stream(\"tell me a 50 word tale\"):\n", | ||
| " data = json.loads(json.loads(chunk)[\"data\"])[\"stream_token\"]\n", | ||
| " print(data, end=\"\", flush=True)" | ||
| ] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "peenv", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.9.13" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 2 | ||
| } |