diff --git a/packages/workspace/src/core/file-utils.ts b/packages/workspace/src/core/file-utils.ts index 5954342e707cd..8b14b474cf8e3 100644 --- a/packages/workspace/src/core/file-utils.ts +++ b/packages/workspace/src/core/file-utils.ts @@ -280,7 +280,7 @@ export function readWorkspaceFiles(projectGraphVersion = '3.0'): FileData[] { if (defaultFileHasher.usesGitForHashing) { const ignoredGlobs = getIgnoredGlobs(); - const r = defaultFileHasher.workspaceFiles + const r = Array.from(defaultFileHasher.workspaceFiles) .filter((f) => !ignoredGlobs.ignores(f)) .map((f) => projectFileDataCompatAdapter( diff --git a/packages/workspace/src/core/hasher/file-hasher.ts b/packages/workspace/src/core/hasher/file-hasher.ts index f4a0d9c1f9cf9..8b64fdff656fb 100644 --- a/packages/workspace/src/core/hasher/file-hasher.ts +++ b/packages/workspace/src/core/hasher/file-hasher.ts @@ -1,11 +1,14 @@ import { appRootPath } from '@nrwl/tao/src/utils/app-root'; import { performance } from 'perf_hooks'; -import { getFileHashes } from './git-hasher'; +import { + getFileHashes, + getUntrackedAndUncommittedFileHashes, +} from './git-hasher'; import { defaultHashing, HashingImpl } from './hashing-impl'; export class FileHasher { fileHashes: { [path: string]: string } = {}; - workspaceFiles: string[] = []; + workspaceFiles = new Set(); usesGitForHashing = false; private isInitialized = false; @@ -13,7 +16,7 @@ export class FileHasher { clear(): void { this.fileHashes = {}; - this.workspaceFiles = []; + this.workspaceFiles = new Set(); this.usesGitForHashing = false; } @@ -31,6 +34,38 @@ export class FileHasher { ); } + /** + * This method is used in cases where we do not want to fully tear down the + * known state of file hashes, and instead only want to hash the currently + * uncommitted (both staged and unstaged) non-deleted files. + * + * For example, the daemon server can cache the last known commit SHA in + * memory and avoid calling init() by using this method instead when that + * SHA is unchanged. + */ + incrementalUpdate() { + performance.mark('incremental hashing:start'); + + const untrackedAndUncommittedFileHashes = + getUntrackedAndUncommittedFileHashes(appRootPath); + + untrackedAndUncommittedFileHashes.forEach((hash, filename) => { + this.fileHashes[filename] = hash; + /** + * we have to store it separately because fileHashes can be modified + * later on and can contain files that do not exist in the workspace + */ + this.workspaceFiles.add(filename); + }); + + performance.mark('incremental hashing:end'); + performance.measure( + 'incremental hashing', + 'incremental hashing:start', + 'incremental hashing:end' + ); + } + hashFile(path: string): string { this.ensureInitialized(); @@ -57,7 +92,7 @@ export class FileHasher { * we have to store it separately because fileHashes can be modified * later on and can contain files that do not exist in the workspace */ - this.workspaceFiles.push(filename.substr(sliceIndex)); + this.workspaceFiles.add(filename.substr(sliceIndex)); }); } diff --git a/packages/workspace/src/core/hasher/git-hasher.ts b/packages/workspace/src/core/hasher/git-hasher.ts index 617009fdf8fb0..3e19941b785d6 100644 --- a/packages/workspace/src/core/hasher/git-hasher.ts +++ b/packages/workspace/src/core/hasher/git-hasher.ts @@ -94,6 +94,10 @@ function gitLsTree(path: string): Map { ); } +export function gitRevParseHead(path: string): string { + return spawnProcess('git', ['rev-parse', 'HEAD'], path); +} + function gitStatus(path: string): { status: Map; deletedFiles: string[]; @@ -166,3 +170,17 @@ export function getFileHashes(path: string): Map { return new Map(); } } + +/** + * This utility is used to return a Map of filenames to hashes, where those filenames come from + * git's knowledge of: + * + * - files which are untracked (newly created) + * - files which are modified in some way (but NOT deleted) and either staged or unstaged + */ +export function getUntrackedAndUncommittedFileHashes( + path: string +): Map { + const { status } = gitStatus(path); + return status; +} diff --git a/packages/workspace/src/core/project-graph/daemon/server.ts b/packages/workspace/src/core/project-graph/daemon/server.ts index c29d8a9a24787..911726a90ef56 100644 --- a/packages/workspace/src/core/project-graph/daemon/server.ts +++ b/packages/workspace/src/core/project-graph/daemon/server.ts @@ -6,6 +6,7 @@ import { platform } from 'os'; import { join, resolve } from 'path'; import { performance, PerformanceObserver } from 'perf_hooks'; import { defaultFileHasher } from '../../hasher/file-hasher'; +import { gitRevParseHead } from '../../hasher/git-hasher'; import { createProjectGraph } from '../project-graph'; /** @@ -64,6 +65,13 @@ function formatLogMessage(message) { return `[NX Daemon Server] - ${new Date().toISOString()} - ${message}`; } +/** + * We cache the latest known HEAD value on the server so that we can potentially skip + * some work initializing file hashes. If the HEAD value has not changed since we last + * initialized the hashes, then we can move straight on to hashing uncommitted changes. + */ +let cachedGitHead: string | undefined; + /** * For now we just invoke the existing `createProjectGraph()` utility and return the project * graph upon connection to the server @@ -82,7 +90,13 @@ const server = createServer((socket) => { performance.mark('server-connection'); serverLog('Connection Received'); - defaultFileHasher.init(); + const currentGitHead = gitRevParseHead(appRootPath); + if (currentGitHead === cachedGitHead) { + defaultFileHasher.incrementalUpdate(); + } else { + defaultFileHasher.init(); + cachedGitHead = currentGitHead; + } const projectGraph = createProjectGraph( undefined,