Merge branch 'main' into 18-handle-errors-given-from-api-responses

Cargo.toml

@@ -23,8 +23,8 @@ features = ["derive"]
 [dependencies.openai_proc_macros]
 path = "openai_proc_macros"
 
-[dependencies.openai_utils]
-path = "openai_utils"
+[dependencies.openai_bootstrap]
+path = "openai_bootstrap"
 
 [dev-dependencies.tokio]
 version = "1.25"

openai_utils/Cargo.toml → openai_bootstrap/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "openai_utils"
+name = "openai_bootstrap"
 version.workspace = true
 authors.workspace = true
 edition.workspace = true

openai_bootstrap/src/lib.rs

@@ -0,0 +1,17 @@
+pub const BASE_URL: &str = "https://api.openai.com/v1";
+
+#[macro_export]
+macro_rules! authorization {
+    ($request:expr) => {{
+        use dotenvy::dotenv;
+        use reqwest::{header::AUTHORIZATION, RequestBuilder};
+        use std::env;
+
+        dotenv().ok();
+
+        let token =
+            env::var("OPENAI_KEY").expect("environment variable `OPENAI_KEY` should be defined");
+
+        $request.header(AUTHORIZATION, format!("Bearer {token}"))
+    }};
+}

openai_proc_macros/Cargo.toml

@@ -22,8 +22,8 @@ convert_case = "0.6"
 version = "0.11"
 features = [ "blocking", "json" ]
 
-[dependencies.openai_utils]
-path = "../openai_utils"
+[dependencies.openai_bootstrap]
+path = "../openai_bootstrap"
 
 [dependencies.serde]
 version = "1.0"

openai_proc_macros/src/lib.rs

@@ -1,9 +1,9 @@
+use convert_case::{Case, Casing};
+use openai_bootstrap::{authorization, BASE_URL};
 use proc_macro::TokenStream;
-use quote::{ quote, format_ident };
+use quote::{format_ident, quote};
 use reqwest::blocking::Client;
-use openai_utils::{ BASE_URL, authorization };
 use serde::Deserialize;
-use convert_case::{ Case, Casing };
 
 #[derive(Deserialize)]
 struct Models {
@@ -20,18 +20,18 @@ pub fn generate_model_id_enum(_input: TokenStream) -> TokenStream {
     let client = Client::new();
 
     let response: Models = authorization!(client.get(format!("{BASE_URL}/models")))
-        .send().unwrap().json().unwrap();
+        .send()
+        .unwrap()
+        .json()
+        .unwrap();
 
     let mut model_id_idents = Vec::new();
     let mut model_ids = Vec::new();
     let mut model_indexes = Vec::new();
     let mut index: u32 = 0;
 
     for model in response.data {
-        if model.id.contains(':')
-           || model.id.contains('.')
-           || model.id.contains("deprecated")
-        {
+        if model.id.contains(':') || model.id.contains('.') || model.id.contains("deprecated") {
             continue;
         }
 

src/completions.rs

@@ -2,7 +2,7 @@
 //! and can also return the probabilities of alternative tokens at each position.
 
 use super::{handle_api, models::ModelID, ModifiedApiResponse, Usage};
-use openai_utils::{authorization, BASE_URL};
+use openai_bootstrap::{ BASE_URL, authorization };
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;

src/edits.rs

@@ -1,7 +1,7 @@
 //! Given a prompt and an instruction, the model will return an edited version of the prompt.
 
 use super::{handle_api, models::ModelID, ModifiedApiResponse, OpenAiError, Usage};
-use openai_utils::{authorization, BASE_URL};
+use openai_bootstrap::{ BASE_URL, authorization };
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 
@@ -91,6 +91,6 @@ mod tests {
         assert_eq!(
             edit.choices.first().unwrap(),
             "What day of the week is it?\n"
-        )
+        );
     }
 }

src/embeddings.rs

@@ -3,7 +3,7 @@
 //! Related guide: [Embeddings](https://beta.openai.com/docs/guides/embeddings)
 
 use super::{handle_api, models::ModelID, ModifiedApiResponse, Usage};
-use openai_utils::{authorization, BASE_URL};
+use openai_bootstrap::{authorization, BASE_URL};
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 
@@ -43,12 +43,27 @@ impl Embeddings {
 
         handle_api(request).await
     }
+
+    pub fn distances(&self) -> Vec<f64> {
+        let mut distances = Vec::new();
+        let mut last_embedding: Option<&Embedding> = None;
+
+        for embedding in &self.data {
+            if let Some(other) = last_embedding {
+                distances.push(embedding.distance(other));
+            }
+
+            last_embedding = Some(embedding);
+        }
+
+        distances
+    }
 }
 
 #[derive(Deserialize)]
 pub struct Embedding {
     #[serde(rename = "embedding")]
-    pub vec: Vec<f32>,
+    pub vec: Vec<f64>,
 }
 
 impl Embedding {
@@ -60,6 +75,18 @@ impl Embedding {
             Err(error) => Ok(Err(error)),
         }
     }
+
+    pub fn distance(&self, other: &Self) -> f64 {
+        let dot_product: f64 = self
+            .vec
+            .iter()
+            .zip(other.vec.iter())
+            .map(|(x, y)| x * y)
+            .sum();
+        let product_of_lengths = (self.vec.len() * other.vec.len()) as f64;
+
+        dot_product / product_of_lengths
+    }
 }
 
 #[cfg(test)]
@@ -80,7 +107,7 @@ mod tests {
         .unwrap()
         .unwrap();
 
-        assert!(!embeddings.data.first().unwrap().vec.is_empty())
+        assert!(!embeddings.data.first().unwrap().vec.is_empty());
     }
 
     #[tokio::test]
@@ -96,6 +123,50 @@ mod tests {
         .unwrap()
         .unwrap();
 
-        assert!(!embedding.vec.is_empty())
+        assert!(!embedding.vec.is_empty());
+    }
+
+    #[test]
+    fn right_angle() {
+        let embeddings = Embeddings {
+            data: vec![
+                Embedding {
+                    vec: vec![1.0, 0.0, 0.0],
+                },
+                Embedding {
+                    vec: vec![0.0, 1.0, 0.0],
+                },
+            ],
+            model: ModelID::TextEmbeddingAda002,
+            usage: Usage {
+                prompt_tokens: 0,
+                completion_tokens: Some(0),
+                total_tokens: 0,
+            },
+        };
+
+        assert_eq!(embeddings.distances()[0], 0.0);
+    }
+
+    #[test]
+    fn non_right_angle() {
+        let embeddings = Embeddings {
+            data: vec![
+                Embedding {
+                    vec: vec![1.0, 1.0, 0.0],
+                },
+                Embedding {
+                    vec: vec![0.0, 1.0, 0.0],
+                },
+            ],
+            model: ModelID::TextEmbeddingAda002,
+            usage: Usage {
+                prompt_tokens: 0,
+                completion_tokens: Some(0),
+                total_tokens: 0,
+            },
+        };
+
+        assert_ne!(embeddings.distances()[0], 0.0);
     }
 }

src/models.rs

@@ -4,7 +4,7 @@
 
 use super::{handle_api, ModifiedApiResponse};
 use openai_proc_macros::generate_model_id_enum;
-use openai_utils::{authorization, BASE_URL};
+use openai_bootstrap::{ BASE_URL, authorization };
 use reqwest::Client;
 use serde::Deserialize;
 
@@ -99,7 +99,10 @@ mod tests {
 
         let model = Model::new(ModelID::TextDavinci003).await.unwrap().unwrap();
 
-        assert_eq!(model.id, ModelID::TextDavinci003,)
+        assert_eq!(
+            model.id,
+            ModelID::TextDavinci003,
+        );
     }
 
     #[tokio::test]
@@ -116,6 +119,6 @@ mod tests {
         assert_eq!(
             model.id,
             ModelID::Custom("davinci:ft-personal-2022-12-12-04-49-51".to_string()),
-        )
+        );
     }
 }