mono repo structure

vkrit · May 1, 2023 · 02d1bdb · 02d1bdb
1 parent b00d338
commit 02d1bdb
Show file tree

Hide file tree

Showing 53 changed files with 20 additions and 254 deletions.
diff --git a/.gitignore b/.gitignore
@@ -169,4 +169,6 @@ cython_debug/
 
 # vs code
 .vscode
-*.bin
+*.bin
+
+.DS_Store
diff --git a/eval_data/user_oriented_instructions.jsonl b/eval_data/user_oriented_instructions.jsonl
diff --git a/gpt4all-api/README.md b/gpt4all-api/README.md
@@ -0,0 +1,2 @@
+# GPT4All API
+This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.
diff --git a/gpt4all-backend/README.md b/gpt4all-backend/README.md
@@ -0,0 +1,2 @@
+# GPT4ALL Backend
+This directory will contain the C/C++ model backends. We will want a subdirectory for each model we build out (e.g. gptj, llama). Ideally, there will be a universal library/wrapper for all models. Language bindings will be build on top of the universal library.
diff --git a/gpt4all-backend/gptj/placeholder b/gpt4all-backend/gptj/placeholder
diff --git a/gpt4all-backend/llama/placeholder b/gpt4all-backend/llama/placeholder
diff --git a/gpt4all-bindings/README.md b/gpt4all-bindings/README.md
@@ -0,0 +1,3 @@
+# GPT4All Bindings
+This directory will contain language specific bindings on top of the C/C++ model backends.
+We will have one directory per language binding (e.g. Python, Typescript, Go, etc.).
diff --git a/gpt4all-bindings/python/placeholder b/gpt4all-bindings/python/placeholder
diff --git a/gpt4all-bindings/typescript/placeholder b/gpt4all-bindings/typescript/placeholder
diff --git a/gpt4all-chat/README.md b/gpt4all-chat/README.md
@@ -0,0 +1,2 @@
+# GPT4All Chat
+This directory will contain the code to build out the QT chat GUI.
diff --git a/gpt4all-docker/README.md b/gpt4all-docker/README.md
@@ -0,0 +1,8 @@
+# GPT4All Docker
+This directory will contain Dockerfiles to build out different gpt4all recipes.
+
+For example:
+1. Docker container that builds out gpt4all RESTful API.
+2. Docker container that builds out gpt4all model backends and Python bindings.
+3. Docker container that builds out everything.
+4. etc.
diff --git a/GPT-J_MAP.md → gpt4all-training/GPT-J_MAP.md b/GPT-J_MAP.md → gpt4all-training/GPT-J_MAP.md
diff --git a/TRAINING_LOG.md → gpt4all-training/TRAINING_LOG.md b/TRAINING_LOG.md → gpt4all-training/TRAINING_LOG.md
diff --git a/build_map.py → gpt4all-training/build_map.py b/build_map.py → gpt4all-training/build_map.py
diff --git a/chat/gpt4all-lora-quantized-OSX-intel → ...ing/chat/gpt4all-lora-quantized-OSX-intel b/chat/gpt4all-lora-quantized-OSX-intel → ...ing/chat/gpt4all-lora-quantized-OSX-intel
diff --git a/chat/gpt4all-lora-quantized-OSX-m1 → ...aining/chat/gpt4all-lora-quantized-OSX-m1 b/chat/gpt4all-lora-quantized-OSX-m1 → ...aining/chat/gpt4all-lora-quantized-OSX-m1
diff --git a/chat/gpt4all-lora-quantized-linux-x86 → ...ing/chat/gpt4all-lora-quantized-linux-x86 b/chat/gpt4all-lora-quantized-linux-x86 → ...ing/chat/gpt4all-lora-quantized-linux-x86
diff --git a/chat/gpt4all-lora-quantized-win64.exe → ...ing/chat/gpt4all-lora-quantized-win64.exe b/chat/gpt4all-lora-quantized-win64.exe → ...ing/chat/gpt4all-lora-quantized-win64.exe
diff --git a/clean.py → gpt4all-training/clean.py b/clean.py → gpt4all-training/clean.py
diff --git a/configs/deepspeed/ds_config.json → ...training/configs/deepspeed/ds_config.json b/configs/deepspeed/ds_config.json → ...training/configs/deepspeed/ds_config.json
diff --git a/configs/deepspeed/ds_config_gptj.json → ...ing/configs/deepspeed/ds_config_gptj.json b/configs/deepspeed/ds_config_gptj.json → ...ing/configs/deepspeed/ds_config_gptj.json
diff --git a/configs/deepspeed/ds_config_gptj_lora.json → ...onfigs/deepspeed/ds_config_gptj_lora.json b/configs/deepspeed/ds_config_gptj_lora.json → ...onfigs/deepspeed/ds_config_gptj_lora.json
diff --git a/configs/eval/generate_baseline.yaml → ...ining/configs/eval/generate_baseline.yaml b/configs/eval/generate_baseline.yaml → ...ining/configs/eval/generate_baseline.yaml
diff --git a/configs/eval/generate_gpt4all_gptj.yaml → ...g/configs/eval/generate_gpt4all_gptj.yaml b/configs/eval/generate_gpt4all_gptj.yaml → ...g/configs/eval/generate_gpt4all_gptj.yaml
diff --git a/configs/eval/generate_gpt4all_gptj_lora.yaml → ...figs/eval/generate_gpt4all_gptj_lora.yaml b/configs/eval/generate_gpt4all_gptj_lora.yaml → ...figs/eval/generate_gpt4all_gptj_lora.yaml
diff --git a/...igs/eval/generate_gpt4all_llama_lora.yaml → ...igs/eval/generate_gpt4all_llama_lora.yaml b/...igs/eval/generate_gpt4all_llama_lora.yaml → ...igs/eval/generate_gpt4all_llama_lora.yaml
diff --git a/configs/generate/generate.yaml → ...l-training/configs/generate/generate.yaml b/configs/generate/generate.yaml → ...l-training/configs/generate/generate.yaml
diff --git a/configs/generate/generate_gptj.yaml → ...ining/configs/generate/generate_gptj.yaml b/configs/generate/generate_gptj.yaml → ...ining/configs/generate/generate_gptj.yaml
diff --git a/configs/generate/generate_gptj_lora.yaml → .../configs/generate/generate_gptj_lora.yaml b/configs/generate/generate_gptj_lora.yaml → .../configs/generate/generate_gptj_lora.yaml
diff --git a/configs/generate/generate_llama.yaml → ...ning/configs/generate/generate_llama.yaml b/configs/generate/generate_llama.yaml → ...ning/configs/generate/generate_llama.yaml
diff --git a/configs/inference/gptj.yaml → gpt4all-training/configs/inference/gptj.yaml b/configs/inference/gptj.yaml → gpt4all-training/configs/inference/gptj.yaml
diff --git a/configs/train/finetune.yaml → gpt4all-training/configs/train/finetune.yaml b/configs/train/finetune.yaml → gpt4all-training/configs/train/finetune.yaml
diff --git a/configs/train/finetune_gptj.yaml → ...training/configs/train/finetune_gptj.yaml b/configs/train/finetune_gptj.yaml → ...training/configs/train/finetune_gptj.yaml
diff --git a/configs/train/finetune_gptj_lora.yaml → ...ing/configs/train/finetune_gptj_lora.yaml b/configs/train/finetune_gptj_lora.yaml → ...ing/configs/train/finetune_gptj_lora.yaml
diff --git a/configs/train/finetune_lora.yaml → ...training/configs/train/finetune_lora.yaml b/configs/train/finetune_lora.yaml → ...training/configs/train/finetune_lora.yaml
diff --git a/create_hostname.sh → gpt4all-training/create_hostname.sh b/create_hostname.sh → gpt4all-training/create_hostname.sh
diff --git a/data.py → gpt4all-training/data.py b/data.py → gpt4all-training/data.py
diff --git a/env.yaml → gpt4all-training/env.yaml b/env.yaml → gpt4all-training/env.yaml
diff --git a/eval_figures.py → gpt4all-training/eval_figures.py b/eval_figures.py → gpt4all-training/eval_figures.py
diff --git a/eval_self_instruct.py → gpt4all-training/eval_self_instruct.py b/eval_self_instruct.py → gpt4all-training/eval_self_instruct.py
diff --git a/figs/clustering_overfit.png → gpt4all-training/figs/clustering_overfit.png b/figs/clustering_overfit.png → gpt4all-training/figs/clustering_overfit.png
diff --git a/figs/duplicate_loss.png → gpt4all-training/figs/duplicate_loss.png b/figs/duplicate_loss.png → gpt4all-training/figs/duplicate_loss.png
diff --git a/figs/first_lora.png → gpt4all-training/figs/first_lora.png b/figs/first_lora.png → gpt4all-training/figs/first_lora.png
diff --git a/figs/overfit-gpt-j.png → gpt4all-training/figs/overfit-gpt-j.png b/figs/overfit-gpt-j.png → gpt4all-training/figs/overfit-gpt-j.png
diff --git a/figs/perplexity_hist.png → gpt4all-training/figs/perplexity_hist.png b/figs/perplexity_hist.png → gpt4all-training/figs/perplexity_hist.png
diff --git a/figs/single_epoch.png → gpt4all-training/figs/single_epoch.png b/figs/single_epoch.png → gpt4all-training/figs/single_epoch.png
diff --git a/generate.py → gpt4all-training/generate.py b/generate.py → gpt4all-training/generate.py
diff --git a/inference.py → gpt4all-training/inference.py b/inference.py → gpt4all-training/inference.py
diff --git a/launcher.sh → gpt4all-training/launcher.sh b/launcher.sh → gpt4all-training/launcher.sh
diff --git a/read.py → gpt4all-training/read.py b/read.py → gpt4all-training/read.py
diff --git a/requirements.txt → gpt4all-training/requirements.txt b/requirements.txt → gpt4all-training/requirements.txt
diff --git a/train.py → gpt4all-training/train.py b/train.py → gpt4all-training/train.py
diff --git a/peft b/peft
-Original file line number
+Diff line change
@@ Expand Up / @@ -169,4 +169,6 @@ cython_debug/ @@
     # vs code
     .vscode
-    *.bin
+    *.bin
+    .DS_Store
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# GPT4All API
		This directory will contain code to build out a RESTful API for GPT4All models. Exact details TBD, but as an MVP, user should be able to send requests to list, download, and generate text with different models.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# GPT4ALL Backend
		This directory will contain the C/C++ model backends. We will want a subdirectory for each model we build out (e.g. gptj, llama). Ideally, there will be a universal library/wrapper for all models. Language bindings will be build on top of the universal library.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# GPT4All Chat
		This directory will contain the code to build out the QT chat GUI.