Construct & render the full gpt layout

oliveiracwb · Feb 25, 2023 · b874be7 · b874be7
1 parent a86dec2
commit b874be7
Show file tree

Hide file tree

Showing 7 changed files with 328 additions and 68 deletions.
diff --git a/progress/progress-20230225-full-model-layout.png b/progress/progress-20230225-full-model-layout.png
diff --git a/progress/progress-20230225-multihead-self-attention.png b/progress/progress-20230225-multihead-self-attention.png
diff --git a/src/GptModel.ts b/src/GptModel.ts
@@ -193,7 +193,7 @@ export function createAttnLayer(layerBuilder: ILayerBuilder, prefix: string, inp
                 m = max(m, p);
             }
 
-            // Pass 2 finds the sum (shifted by max)
+            // Pass 2 finds the exp sum (shifted by max)
             float a = 0.0;
             for (int i = 0; i <= tIdxY; i++) {
                 float p = texelFetch(attnMatrix, ivec2(i, pos.y), 0).r;