switch to venv and create requirements.txt

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .DS_Store
 __pycache__
+venv

README.md

@@ -16,6 +16,18 @@ npm install
 npm run dev
 ```
 
+## Python setup
+The following directories depend on a virtual env
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+## notebooks
+
+
 ## scripts
 Python scripts that can be run via CLI or via the web interface (through the server). The scripts assume a certain directory structure in the data folder.  
 See below for more detailed instructions on using the scripts

requirements.txt

@@ -0,0 +1,52 @@
+blinker==1.7.0
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.0
+cycler==0.12.1
+Cython==0.29.37
+filelock==3.13.1
+Flask==3.0.0
+Flask-Cors==4.0.0
+fonttools==4.47.2
+fsspec==2023.12.2
+hdbscan==0.8.33
+huggingface-hub==0.20.2
+idna==3.6
+itsdangerous==2.1.2
+Jinja2==3.1.3
+joblib==1.3.2
+kiwisolver==1.4.5
+llvmlite==0.41.1
+MarkupSafe==2.1.3
+matplotlib==3.8.2
+mpmath==1.3.0
+networkx==3.2.1
+numba==0.58.1
+numpy==1.26.3
+packaging==23.2
+pandas==2.1.4
+pillow==10.2.0
+pyarrow==14.0.2
+pynndescent==0.5.11
+pyparsing==3.1.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+regex==2023.12.25
+requests==2.31.0
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+six==1.16.0
+sympy==1.12
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.2
+tqdm==4.66.1
+transformers==4.36.2
+typing_extensions==4.9.0
+tzdata==2023.4
+umap-learn==0.5.5
+urllib3==2.1.0
+Werkzeug==3.0.1

scripts/embed.py

@@ -1,7 +1,6 @@
 # Usage: python embed.py <dataset_name> <text_column>
 import os
 import sys
-import time
 import json
 import torch
 import numpy as np