fix(core): filter branch in preparation for nx import (nrwl#27652)

This PR fixes an issue with `nx import` where multiple directories
cannot be imported from the same repo. We now use `git filter-repo` (if
installed) or `git filter-branch` (fallback) to prepare the repo for
import so anything not in the subdirectory will be ignored (i.e. cleaner
history).

Note: `filter-repo` is much faster but requires the user to install it
first via their package manager (e.g. `brew install git-filter-repo` or
`apt install git-filter-repo`). We fallback to `git filter-branch` since
it comes with git, but it is slower, so the process may take 10+ minutes
on really large monorepos (e.g. next.js).

Also:
- Use `await` before returning a promise to Node can maintain correct
stacktrace
- Remove logic for `if (relativeSourceDir === '')` since using
`filter-branch` moves all the files to the root (`.`)
- Default destination project location to be the same as source (e.g.
importing `packages/a` will go to `packages/a` unless user types in
something else)
- Add `--depth` option if users don't want to clone with full history
(default is full history)
- Fix issues with special characters causing `git ls-files` + `git mv`
to since `mv` doesn't work with escaped names

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->

## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

Author: jaysoo

e2e/nx/src/import.test.ts

@@ -8,7 +8,7 @@ import {
   updateFile,
   e2eCwd,
 } from '@nx/e2e/utils';
-import { mkdirSync, rmdirSync } from 'fs';
+import { writeFileSync, mkdirSync, rmdirSync } from 'fs';
 import { execSync } from 'node:child_process';
 import { join } from 'path';
 
@@ -86,4 +86,54 @@ describe('Nx Import', () => {
     );
     runCLI(`vite:build created-vite-app`);
   });
+
+  it('should be able to import two directories from same repo', () => {
+    // Setup repo with two packages: a and b
+    const repoPath = join(tempImportE2ERoot, 'repo');
+    mkdirSync(repoPath, { recursive: true });
+    writeFileSync(join(repoPath, 'README.md'), `# Repo`);
+    execSync(`git init`, {
+      cwd: repoPath,
+    });
+    execSync(`git add .`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -am "initial commit"`, {
+      cwd: repoPath,
+    });
+    execSync(`git checkout -b main`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/a'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/a/README.md'), `# A`);
+    execSync(`git add packages/a`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package a"`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/b'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/b/README.md'), `# B`);
+    execSync(`git add packages/b`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package b"`, {
+      cwd: repoPath,
+    });
+
+    runCLI(
+      `import ${repoPath} packages/a --ref main --source packages/a --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+    runCLI(
+      `import ${repoPath} packages/b --ref main --source packages/b --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+
+    checkFilesExist('packages/a/README.md', 'packages/b/README.md');
+  });
 });

packages/nx/src/command-line/import/command-object.ts

@@ -28,6 +28,11 @@ export const yargsImportCommand: CommandModule = {
             type: 'string',
             description: 'The branch from the source repository to import',
           })
+          .option('depth', {
+            type: 'number',
+            description:
+              'The depth to clone the source repository (limit this for faster git clone)',
+          })
           .option('interactive', {
             type: 'boolean',
             description: 'Interactive mode',

packages/nx/src/command-line/import/import.ts

@@ -1,4 +1,8 @@
-import { join, relative, resolve } from 'path';
+import { dirname, join, relative, resolve } from 'path';
+import { minimatch } from 'minimatch';
+import { existsSync, promises as fsp } from 'node:fs';
+import * as chalk from 'chalk';
+import { load as yamlLoad } from '@zkochan/js-yaml';
 import { cloneFromUpstream, GitRepository } from '../../utils/git-utils';
 import { stat, mkdir, rm } from 'node:fs/promises';
 import { tmpdir } from 'tmp';
@@ -11,6 +15,9 @@ import { workspaceRoot } from '../../utils/workspace-root';
 import {
   detectPackageManager,
   getPackageManagerCommand,
+  isWorkspacesEnabled,
+  PackageManager,
+  PackageManagerCommands,
 } from '../../utils/package-manager';
 import { resetWorkspaceContext } from '../../utils/workspace-context';
 import { runInstall } from '../init/implementation/utils';
@@ -21,6 +28,7 @@ import {
   getPackagesInPackageManagerWorkspace,
   needsInstall,
 } from './utils/needs-install';
+import { readPackageJson } from '../../project-graph/file-utils';
 
 const importRemoteName = '__tmp_nx_import__';
 
@@ -41,6 +49,10 @@ export interface ImportOptions {
    * The directory in the destination repo to import into
    */
   destination: string;
+  /**
+   * The depth to clone the source repository (limit this for faster clone times)
+   */
+  depth: number;
 
   verbose: boolean;
   interactive: boolean;
@@ -90,7 +102,7 @@ export async function importHandler(options: ImportOptions) {
 
   const sourceRepoPath = join(tempImportDirectory, 'repo');
   const spinner = createSpinner(
-    `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath}`
+    `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath} (Use --depth to limit commit history and speed up clone times)`
   ).start();
   try {
     await rm(tempImportDirectory, { recursive: true });
@@ -101,6 +113,7 @@ export async function importHandler(options: ImportOptions) {
   try {
     sourceGitClient = await cloneFromUpstream(sourceRemoteUrl, sourceRepoPath, {
       originName: importRemoteName,
+      depth: options.depth,
     });
   } catch (e) {
     spinner.fail(`Failed to clone ${sourceRemoteUrl} into ${sourceRepoPath}`);
@@ -110,6 +123,9 @@ export async function importHandler(options: ImportOptions) {
   }
   spinner.succeed(`Cloned into ${sourceRepoPath}`);
 
+  // Detecting the package manager before preparing the source repo for import.
+  const sourcePackageManager = detectPackageManager(sourceGitClient.root);
+
   if (!ref) {
     const branchChoices = await sourceGitClient.listBranches();
     ref = (
@@ -149,6 +165,7 @@ export async function importHandler(options: ImportOptions) {
           name: 'destination',
           message: 'Where in this workspace should the code be imported into?',
           required: true,
+          initial: source ? source : undefined,
         },
       ])
     ).destination;
@@ -157,6 +174,23 @@ export async function importHandler(options: ImportOptions) {
   const absSource = join(sourceRepoPath, source);
   const absDestination = join(process.cwd(), destination);
 
+  const destinationGitClient = new GitRepository(process.cwd());
+  await assertDestinationEmpty(destinationGitClient, absDestination);
+
+  const tempImportBranch = getTempImportBranch(ref);
+  await sourceGitClient.addFetchRemote(importRemoteName, ref);
+  await sourceGitClient.fetch(importRemoteName, ref);
+  spinner.succeed(`Fetched ${ref} from ${sourceRemoteUrl}`);
+  spinner.start(
+    `Checking out a temporary branch, ${tempImportBranch} based on ${ref}`
+  );
+  await sourceGitClient.checkout(tempImportBranch, {
+    new: true,
+    base: `${importRemoteName}/${ref}`,
+  });
+
+  spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`);
+
   try {
     await stat(absSource);
   } catch (e) {
@@ -165,11 +199,6 @@ export async function importHandler(options: ImportOptions) {
     );
   }
 
-  const destinationGitClient = new GitRepository(process.cwd());
-  await assertDestinationEmpty(destinationGitClient, absDestination);
-
-  const tempImportBranch = getTempImportBranch(ref);
-
   const packageManager = detectPackageManager(workspaceRoot);
 
   const originalPackageWorkspaces = await getPackagesInPackageManagerWorkspace(
@@ -186,8 +215,7 @@ export async function importHandler(options: ImportOptions) {
     source,
     relativeDestination,
     tempImportBranch,
-    sourceRemoteUrl,
-    importRemoteName
+    sourceRemoteUrl
   );
 
   await createTemporaryRemote(
@@ -220,22 +248,74 @@ export async function importHandler(options: ImportOptions) {
     options.interactive
   );
 
-  if (plugins.length > 0) {
-    output.log({ title: 'Installing Plugins' });
-    installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts);
+  if (packageManager !== sourcePackageManager) {
+    output.warn({
+      title: `Mismatched package managers`,
+      bodyLines: [
+        `The source repository is using a different package manager (${sourcePackageManager}) than this workspace (${packageManager}).`,
+        `This could lead to install issues due to discrepancies in "package.json" features.`,
+      ],
+    });
+  }
 
-    await destinationGitClient.amendCommit();
+  // If install fails, we should continue since the errors could be resolved later.
+  let installFailed = false;
+  if (plugins.length > 0) {
+    try {
+      output.log({ title: 'Installing Plugins' });
+      installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts);
+
+      await destinationGitClient.amendCommit();
+    } catch (e) {
+      installFailed = true;
+      output.error({
+        title: `Install failed: ${e.message || 'Unknown error'}`,
+        bodyLines: [e.stack],
+      });
+    }
   } else if (await needsInstall(packageManager, originalPackageWorkspaces)) {
-    output.log({
-      title: 'Installing dependencies for imported code',
-    });
+    try {
+      output.log({
+        title: 'Installing dependencies for imported code',
+      });
+
+      runInstall(workspaceRoot, getPackageManagerCommand(packageManager));
+
+      await destinationGitClient.amendCommit();
+    } catch (e) {
+      installFailed = true;
+      output.error({
+        title: `Install failed: ${e.message || 'Unknown error'}`,
+        bodyLines: [e.stack],
+      });
+    }
+  }
 
-    runInstall(workspaceRoot, getPackageManagerCommand(packageManager));
+  console.log(await destinationGitClient.showStat());
 
-    await destinationGitClient.amendCommit();
+  if (installFailed) {
+    const pmc = getPackageManagerCommand(packageManager);
+    output.warn({
+      title: `The import was successful, but the install failed`,
+      bodyLines: [
+        `You may need to run "${pmc.install}" manually to resolve the issue. The error is logged above.`,
+      ],
+    });
   }
 
-  console.log(await destinationGitClient.showStat());
+  await warnOnMissingWorkspacesEntry(packageManager, pmc, relativeDestination);
+
+  // When only a subdirectory is imported, there might be devDependencies in the root package.json file
+  // that needs to be ported over as well.
+  if (ref) {
+    output.log({
+      title: `Check root dependencies`,
+      bodyLines: [
+        `"dependencies" and "devDependencies" are not imported from the source repository (${sourceRemoteUrl}).`,
+        `You may need to add some of those dependencies to this workspace in order to run tasks successfully.`,
+      ],
+    });
+  }
 
   output.log({
     title: `Merging these changes into ${getBaseRef(nxJson)}`,
@@ -274,3 +354,77 @@ async function createTemporaryRemote(
   await destinationGitClient.addGitRemote(remoteName, sourceRemoteUrl);
   await destinationGitClient.fetch(remoteName);
 }
+
+// If the user imports a project that isn't in NPM/Yarn/PNPM workspaces, then its dependencies
+// will not be installed. We should warn users and provide instructions on how to fix this.
+async function warnOnMissingWorkspacesEntry(
+  pm: PackageManager,
+  pmc: PackageManagerCommands,
+  pkgPath: string
+) {
+  if (!isWorkspacesEnabled(pm, workspaceRoot)) {
+    output.warn({
+      title: `Missing workspaces in package.json`,
+      bodyLines:
+        pm === 'npm'
+          ? [
+              `We recommend enabling NPM workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://docs.npmjs.com/cli/using-npm/workspaces`,
+            ]
+          : pm === 'yarn'
+          ? [
+              `We recommend enabling Yarn workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://yarnpkg.com/features/workspaces`,
+            ]
+          : pm === 'bun'
+          ? [
+              `We recommend enabling Bun workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://bun.sh/docs/install/workspaces`,
+            ]
+          : [
+              `We recommend enabling PNPM workspaces to install dependencies for the imported project.`,
+              `Add the following entry to to pnpm-workspace.yaml and run "${pmc.install}":`,
+              chalk.bold(`packages:\n  - '${pkgPath}'`),
+              `See: https://pnpm.io/workspaces`,
+            ],
+    });
+  } else {
+    // Check if the new package is included in existing workspaces entries. If not, warn the user.
+    let workspaces: string[] | null = null;
+
+    if (pm === 'npm' || pm === 'yarn' || pm === 'bun') {
+      const packageJson = readPackageJson();
+      workspaces = packageJson.workspaces;
+    } else if (pm === 'pnpm') {
+      const yamlPath = join(workspaceRoot, 'pnpm-workspace.yaml');
+      if (existsSync(yamlPath)) {
+        const yamlContent = await fsp.readFile(yamlPath, 'utf-8');
+        const yaml = yamlLoad(yamlContent);
+        workspaces = yaml.packages;
+      }
+    }
+
+    if (workspaces) {
+      const isPkgIncluded = workspaces.some((w) => minimatch(pkgPath, w));
+      if (!isPkgIncluded) {
+        const pkgsDir = dirname(pkgPath);
+        output.warn({
+          title: `Project missing in workspaces`,
+          bodyLines:
+            pm === 'npm' || pm === 'yarn' || pm === 'bun'
+              ? [
+                  `The imported project (${pkgPath}) is missing the "workspaces" field in package.json.`,
+                  `Add "${pkgsDir}/*" to workspaces run "${pmc.install}".`,
+                ]
+              : [
+                  `The imported project (${pkgPath}) is missing the "packages" field in pnpm-workspaces.yaml.`,
+                  `Add "${pkgsDir}/*" to packages run "${pmc.install}".`,
+                ],
+        });
+      }
+    }
+  }
+}