feat(core): daemon supports incremental file hashing

packages/workspace/src/core/file-utils.ts

@@ -280,7 +280,7 @@ export function readWorkspaceFiles(projectGraphVersion = '3.0'): FileData[] {
 
   if (defaultFileHasher.usesGitForHashing) {
     const ignoredGlobs = getIgnoredGlobs();
-    const r = defaultFileHasher.workspaceFiles
+    const r = Array.from(defaultFileHasher.workspaceFiles)
       .filter((f) => !ignoredGlobs.ignores(f))
       .map((f) =>
         projectFileDataCompatAdapter(

packages/workspace/src/core/hasher/file-hasher.ts

@@ -1,19 +1,22 @@
 import { appRootPath } from '@nrwl/tao/src/utils/app-root';
 import { performance } from 'perf_hooks';
-import { getFileHashes } from './git-hasher';
+import {
+  getFileHashes,
+  getUntrackedAndUncommittedFileHashes,
+} from './git-hasher';
 import { defaultHashing, HashingImpl } from './hashing-impl';
 
 export class FileHasher {
   fileHashes: { [path: string]: string } = {};
-  workspaceFiles: string[] = [];
+  workspaceFiles = new Set<string>();
   usesGitForHashing = false;
   private isInitialized = false;
 
   constructor(private readonly hashing: HashingImpl) {}
 
   clear(): void {
     this.fileHashes = {};
-    this.workspaceFiles = [];
+    this.workspaceFiles = new Set<string>();
     this.usesGitForHashing = false;
   }
 
@@ -31,6 +34,38 @@ export class FileHasher {
     );
   }
 
+  /**
+   * This method is used in cases where we do not want to fully tear down the
+   * known state of file hashes, and instead only want to hash the currently
+   * uncommitted (both staged and unstaged) non-deleted files.
+   *
+   * For example, the daemon server can cache the last known commit SHA in
+   * memory and avoid calling init() by using this method instead when that
+   * SHA is unchanged.
+   */
+  incrementalUpdate() {
+    performance.mark('incremental hashing:start');
+
+    const untrackedAndUncommittedFileHashes =
+      getUntrackedAndUncommittedFileHashes(appRootPath);
+
+    untrackedAndUncommittedFileHashes.forEach((hash, filename) => {
+      this.fileHashes[filename] = hash;
+      /**
+       * we have to store it separately because fileHashes can be modified
+       * later on and can contain files that do not exist in the workspace
+       */
+      this.workspaceFiles.add(filename);
+    });
+
+    performance.mark('incremental hashing:end');
+    performance.measure(
+      'incremental hashing',
+      'incremental hashing:start',
+      'incremental hashing:end'
+    );
+  }
+
   hashFile(path: string): string {
     this.ensureInitialized();
 
@@ -57,7 +92,7 @@ export class FileHasher {
        * we have to store it separately because fileHashes can be modified
        * later on and can contain files that do not exist in the workspace
        */
-      this.workspaceFiles.push(filename.substr(sliceIndex));
+      this.workspaceFiles.add(filename.substr(sliceIndex));
     });
   }
 

packages/workspace/src/core/hasher/git-hasher.ts

@@ -94,6 +94,10 @@ function gitLsTree(path: string): Map<string, string> {
   );
 }
 
+export function gitRevParseHead(path: string): string {
+  return spawnProcess('git', ['rev-parse', 'HEAD'], path);
+}
+
 function gitStatus(path: string): {
   status: Map<string, string>;
   deletedFiles: string[];
@@ -166,3 +170,17 @@ export function getFileHashes(path: string): Map<string, string> {
     return new Map<string, string>();
   }
 }
+
+/**
+ * This utility is used to return a Map of filenames to hashes, where those filenames come from
+ * git's knowledge of:
+ *
+ * - files which are untracked (newly created)
+ * - files which are modified in some way (but NOT deleted) and either staged or unstaged
+ */
+export function getUntrackedAndUncommittedFileHashes(
+  path: string
+): Map<string, string> {
+  const { status } = gitStatus(path);
+  return status;
+}

packages/workspace/src/core/project-graph/daemon/server.ts

@@ -6,6 +6,7 @@ import { platform } from 'os';
 import { join, resolve } from 'path';
 import { performance, PerformanceObserver } from 'perf_hooks';
 import { defaultFileHasher } from '../../hasher/file-hasher';
+import { gitRevParseHead } from '../../hasher/git-hasher';
 import { createProjectGraph } from '../project-graph';
 
 /**
@@ -64,6 +65,13 @@ function formatLogMessage(message) {
   return `[NX Daemon Server] - ${new Date().toISOString()} - ${message}`;
 }
 
+/**
+ * We cache the latest known HEAD value on the server so that we can potentially skip
+ * some work initializing file hashes. If the HEAD value has not changed since we last
+ * initialized the hashes, then we can move straight on to hashing uncommitted changes.
+ */
+let cachedGitHead: string | undefined;
+
 /**
  * For now we just invoke the existing `createProjectGraph()` utility and return the project
  * graph upon connection to the server
@@ -82,7 +90,13 @@ const server = createServer((socket) => {
   performance.mark('server-connection');
   serverLog('Connection Received');
 
-  defaultFileHasher.init();
+  const currentGitHead = gitRevParseHead(appRootPath);
+  if (currentGitHead === cachedGitHead) {
+    defaultFileHasher.incrementalUpdate();
+  } else {
+    defaultFileHasher.init();
+    cachedGitHead = currentGitHead;
+  }
 
   const projectGraph = createProjectGraph(
     undefined,