Working, but doing nothing yet

kevinrobinson · Nov 14, 2019 · 7495ac5 · 7495ac5
1 parent 940d335
commit 7495ac5
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 101 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,11 @@
 # inspectable-machine
 just early hacking so far
 
+# notes
+"we derive CAVs by training a linear classifier between a concept’s examples and random counter examples and then taking the vector orthogonal to the decision boundary"
+- better: we provide random examples
+
+
 ## screenshots
 ### flow
 ![flow](docs/docs-1.png)

diff --git a/data/concepts-project.zip b/data/concepts-project.zip
diff --git a/data/example-generalization-project.zip b/data/example-generalization-project.zip
diff --git a/data/example-model.zip b/data/example-model.zip
diff --git a/data/training-project.zip b/data/training-project.zip
diff --git a/index.html b/index.html
@@ -44,16 +44,40 @@
   <script src="https://unpkg.com/[email protected]/dist/Prando.umd.js"></script>
 
   <div>
-    <div style="margin: 10px; border-radius: 10px; box-shadow: 0px 2px 2px rgba(0,0,0,0.20); background: url(/img/warp-sti.jpg); backround-repeat: no-repeat; padding: 20px;">
-      <h1><a href="/">warping-machine</a></h1>
+    <div style="margin: 10px; border-radius: 10px; box-shadow: 0px 2px 2px rgba(0,0,0,0.20); background: rgb(229,119, 1); padding: 20px;">
+      <h1><a href="/">questionable-machine</a></h1>
     </div>
 
     <div class="VerticalSection" style="display: flex; flex-direction: row;">
 
       <div class="Panel">
         <div class="Panel-content">
-          <h2>1. Image model</h2>
-          <div class="Panel-text">First, load an image model created in <a target="_blank" href="https://teachablemachine.withgoogle.com/train/image">Teachable Machine</a>.  This only accepts zip files uploaded locally.</div>
+          <h2>1. Training dataset</h2>
+          <div class="Panel-text">First, load the project that you used to create your model in <a target="_blank" href="https://teachablemachine.withgoogle.com/train/image">Teachable Machine</a>.</div>
+        </div>
+        <div class="Panel-actions">
+          <div class="Status" id="training-status"></div>
+          <div class="File Button">
+            <div class="File-caption">
+              <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+                <path fill-rule="evenodd" clip-rule="evenodd" d="M12 6H20C21.1 6 22 6.9 22 8V18C22 19.1 21.1 20 20 20H4C2.9 20 2 19.1 2 18L2.01 6C2.01 4.9 2.9 4 4 4H10L12 6ZM4 8V18H20V8H4Z" fill="#888"></path>
+              </svg>
+              Open training project
+            </div>
+            <input id="training-zip-input" class="fileList" type="file" accept="application/zip">
+          </div>
+          <a class="ExampleLink" href="/data/training-project.zip"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+              <path fill-rule="evenodd" clip-rule="evenodd" d="M13 12.17L15.59 9.59L17 11L12 16L7 11L8.41 9.59L11 12.17V4H13V12.17ZM6 15H4V18C4 19.1 4.9 20 6 20H18C19.1 20 20 19.1 20 18V15H18V18H6V15Z" fill="#888"></path>
+            </svg>
+            <span>Download example</span>
+          </a>
+        </div>
+      </div>
+
+      <div class="Panel">
+        <div class="Panel-content">
+          <h2>2. Image model</h2>
+          <div class="Panel-text">Then, load the image model you created in <a target="_blank" href="https://teachablemachine.withgoogle.com/train/image">Teachable Machine</a>.  This only accepts zip files uploaded locally.</div>
         </div>
         <div class="Panel-actions">
           <div class="Status" id="model-status"></div>
@@ -64,9 +88,9 @@ <h2>1. Image model</h2>
               </svg>
               Open model.zip
             </div>
-            <input id="upload-model-zip-input" class="fileList" type="file" accept="application/zip">
+            <input id="model-zip-input" class="fileList" type="file" accept="application/zip">
           </div>
-          <a class="ExampleLink" href="/data/example-model.zip"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+          <a class="ExampleLink" href="/data/model.zip"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
               <path fill-rule="evenodd" clip-rule="evenodd" d="M13 12.17L15.59 9.59L17 11L12 16L7 11L8.41 9.59L11 12.17V4H13V12.17ZM6 15H4V18C4 19.1 4.9 20 6 20H18C19.1 20 20 19.1 20 18V15H18V18H6V15Z" fill="#888"></path>
             </svg>
             <span>Download example</span>
@@ -76,21 +100,21 @@ <h2>1. Image model</h2>
 
       <div class="Panel">
         <div class="Panel-content">
-          <h2>2. Generalization dataset</h2>
-          <div class="Panel-text">Next, load a project from <a target="_blank" href="https://teachablemachine.withgoogle.com/train/image">Teachable Machine</a> that contains different images than what the model was trained on, but labeled with the same class names.</div>
+          <h2>3. Concepts dataset</h2>
+          <div class="Panel-text">Next, load a project from <a target="_blank" href="https://teachablemachine.withgoogle.com/train/image">Teachable Machine</a> that labels concepts where you want to explore bias in your model.</div>
         </div>
         <div class="Panel-actions">
-          <div class="Status" id="generalization-status"></div>
+          <div class="Status" id="concepts-status"></div>
           <div class="File Button">
             <div class="File-caption">
               <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
                 <path fill-rule="evenodd" clip-rule="evenodd" d="M12 6H20C21.1 6 22 6.9 22 8V18C22 19.1 21.1 20 20 20H4C2.9 20 2 19.1 2 18L2.01 6C2.01 4.9 2.9 4 4 4H10L12 6ZM4 8V18H20V8H4Z" fill="#888"></path>
               </svg>
-              Open project.zip
+              Open concepts project
             </div>
-            <input id="upload-generalization-zip-input" class="fileList" type="file" accept="application/zip">  
+            <input id="concepts-zip-input" class="fileList" type="file" accept="application/zip">  
           </div>
-          <a class="ExampleLink" href="/data/example-generalization-project.zip">
+          <a class="ExampleLink" href="/data/concepts-project.zip">
             <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
               <path fill-rule="evenodd" clip-rule="evenodd" d="M13 12.17L15.59 9.59L17 11L12 16L7 11L8.41 9.59L11 12.17V4H13V12.17ZM6 15H4V18C4 19.1 4.9 20 6 20H18C19.1 20 20 19.1 20 18V15H18V18H6V15Z" fill="#888"></path>
             </svg>
@@ -101,21 +125,17 @@ <h2>2. Generalization dataset</h2>
 
       <div class="Panel">
         <div class="Panel-content">
-          <h2>3. Visualize!</h2>
-          <div class="Panel-text">Now you can visualize how well the model did on the generalization set.<br/><br/>Scan through individual classification errors, or compare how the images appear in <a href="https://github.com/tensorflow/tfjs-models/tree/master/mobilenet" target="_blank">MobileNet</a> embedding space versus your model's embedding space.</div>
+          <h2>4. tcav!</h2>
+          <div class="Panel-text">...</div>
         </div>
         <div class="Panel-actions ButtonList">
-          <button disabled="disabled" id="inspect-button" class="Button">Browse accuracy</button>
-          <button disabled="disabled"id="embeddings-button" class="Button">Visualize embedding space</button>
-          <button disabled="disabled" id="embeddings-3d-button" class="Button">Visualize embeddings in 3D</button>
-          <button disabled="disabled" id="facets-button" class="Button">Explore with Facets Dive</button>
+          <button disabled="disabled" id="tcav-button" class="Button">go</button>
         </div>
       </div>
     </div>
 
     <div>
       <div class="Workspace" id="workspace"></div>
-      <div id="generalization-preview"></div>
     </div>
 
     <div id="log"></div>

diff --git a/src/index.css b/src/index.css
@@ -23,7 +23,7 @@ a {
 h1 {
   /*text-shadow: -1px -1px 0 rgb(229,119, 1), 1px -1px 0 rgb(229,119, 1), -1px 1px 0 rgb(229,119, 1), 1px 1px 0 rgb(229,119, 1);*/
   transform: rotate(-1deg);
-  background: rgb(121, 74, 239);
+  background: rgb(216, 76, 110);
   border-radius: 4px;
   box-shadow: 0px 2px 2px rgba(0,0,0,0.20);
   display: inline-block;
@@ -127,7 +127,7 @@ h2 {
   box-shadow: 0px 2px 2px rgba(0,0,0,0.20);
   padding: 20px;
   margin: 10px;
-  width: 280px;
+  width: 240px;
   display: flex;
   flex-direction: column;
   justify-content: space-between;

diff --git a/src/index.js b/src/index.js
@@ -8,8 +8,9 @@ class App {
     this.el = el;
     this.els = els;
     this.state = {
-      project: null,
-      generalization: null
+      training: null,
+      model: null,
+      concepts: null
     };
   }
 
@@ -23,26 +24,22 @@ class App {
   }
 
   render() {
-    // console.log('render');
-    // const pre = document.createElement('pre');
-    // pre.textContent = JSON.stringify(this.state).length;
-    // this.els.log.appendChild(pre);
-    const {generalization, model} = this.state;
+    const {training, model, concepts} = this.state;
 
 
+    if (training) {
+      this.els.trainingStatus.textContent = 'ready!';
+    }
     if (model) {
       this.els.modelStatus.textContent = 'ready!';
     }
-    if (generalization) {
-      this.els.generalizationStatus.textContent = 'ready!';
+    if (concepts) {
+      this.els.conceptsStatus.textContent = 'ready!';
     }
 
     // buttons
-    const canAnalyze = (generalization && model);
-    this.els.embeddingsButton.disabled = !canAnalyze;
-    this.els.embeddings3DButton.disabled = !canAnalyze;
-    this.els.inspectButton.disabled = !canAnalyze;
-    this.els.facetsButton.disabled = !canAnalyze;
+    const canAnalyze = (training && model && concepts);
+    this.els.tcavButton.disabled = !canAnalyze;
   }
 }
 
@@ -660,17 +657,16 @@ function addWaitingEl(el) {
 
 export async function main(deps) {
   const els = {
-    modelZipInput: document.querySelector('#upload-model-zip-input'),
+    trainingZipInput: document.querySelector('#training-zip-input'),
+    trainingStatus: document.querySelector('#training-status'),
+
+    modelZipInput: document.querySelector('#model-zip-input'),
     modelStatus: document.querySelector('#model-status'),
 
-    generalizationStatus: document.querySelector('#generalization-status'),
-    generalizationZipInput: document.querySelector('#upload-generalization-zip-input'),
-    generalizationPreview: document.querySelector('#generalization-preview'),
+    conceptsZipInput: document.querySelector('#concepts-zip-input'),
+    conceptsStatus: document.querySelector('#concepts-status'),
 
-    inspectButton: document.querySelector('#inspect-button'),
-    embeddingsButton: document.querySelector('#embeddings-button'),
-    embeddings3DButton: document.querySelector('#embeddings-3d-button'),
-    facetsButton: document.querySelector('#facets-button'),
+    tcavButton: document.querySelector('#tcav-button'),
 
     workspace: document.querySelector('#workspace'),
     log: document.querySelector('#log')
@@ -680,6 +676,14 @@ export async function main(deps) {
   app.render();
 
 
+  els.trainingZipInput.addEventListener('change', async e => {
+    if (e.target.files.length === 0) return;
+    els.trainingStatus.textContent = 'loading...';
+    const [trainingZip] = e.target.files;
+    const training = await loadImageProjectFromZipFile(trainingZip);
+    app.update({training});
+  });
+
   els.modelZipInput.addEventListener('change', async e => {
     if (e.target.files.length === 0) return;
     els.modelStatus.textContent = 'loading...';
@@ -688,67 +692,18 @@ export async function main(deps) {
     app.update({model});
   });
 
-  els.generalizationZipInput.addEventListener('change', async e => {
+  els.conceptsZipInput.addEventListener('change', async e => {
     if (e.target.files.length === 0) return;
-    els.generalizationStatus.textContent = 'loading...';
-    const [projectZip] = e.target.files;
-    const generalization = await loadImageProjectFromZipFile(projectZip);
-    app.update({generalization});
-
-    const previewGeneralizationImages = false;
-    if (previewGeneralizationImages) {
-      els.generalizationPreview.innerHTML = '';
-      await mapExamples(generalization, async (className, blobUrl, index) => {
-        const imgEl = document.createElement('img');
-        imgEl.src = blobUrl;
-        imgEl.width = 224/4;
-        imgEl.height = 224/4;
-        els.generalizationPreview.appendChild(imgEl);
-      });
-    }
-  });
-
-  els.inspectButton.addEventListener('click', async e => {
-    const {model, generalization} = app.readState();
-    if (!model || !generalization) return;
-    els.workspace.textContent = 'working...';
-    await inspect(generalization, model, els.workspace, deps);
-  });
-
-  els.embeddingsButton.addEventListener('click', async e => {
-    const {model, generalization} = app.readState();
-    if (!model || !generalization) return;
-    els.workspace.textContent = 'working...';
-    await embeddings(generalization, model, els.workspace, {
-      umap: { nComponents: 2 }, 
-      sprites: false,
-      color: true
-    });
+    els.conceptsStatus.textContent = 'loading...';
+    const [conceptsZip] = e.target.files;
+    const concepts = await loadImageProjectFromZipFile(conceptsZip);
+    app.update({concepts});
   });
 
-  els.embeddings3DButton.addEventListener('click', async e => {
-    const {model, generalization} = app.readState();
-    if (!model || !generalization) return;
+  els.tcavButton.addEventListener('click', async e => {
+    const {training, model, concepts} = app.readState();
+    if (!training || !model || !concepts) return;
     els.workspace.textContent = 'working...';
-    await embeddings(generalization, model, els.workspace, {
-      umap: { nComponents: 3 }, 
-      sprites: true,
-      color: false
-    });
-  });
-
-  els.facetsButton.addEventListener('click', async e => {
-    const {model, generalization} = app.readState();
-    if (!model || !generalization) return;
-
-    const examples = await mapExamples(generalization, async (className, blobUrl, index) => {
-      const imgEl = document.createElement('img');
-      await setImageSrc(imgEl, blobUrl);
-      const predictions = await model.predict(imgEl);
-      return {className, index, predictions, blobUrl};
-    });
-    const done = addWaitingEl(els.workspace);
-    await deps.facets(els.workspace, examples);
-    done();
+    await tcav(model, training, concepts, els.workspace, deps);
   });
 }