feat(core): use durations from task history to schedule tasks (nrwl#2…

…7783)

<!-- Please make sure you have read the submission guidelines before
posting an PR -->
<!--
https://github.com/nrwl/nx/blob/master/CONTRIBUTING.md#-submitting-a-pr
-->

<!-- Please make sure that your commit message follows our format -->
<!-- Example: `fix(nx): must begin with lowercase` -->

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->

## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

Co-authored-by: FrozenPandaz <[email protected]>

packages/nx/Cargo.toml

@@ -34,7 +34,7 @@ nom = '7.1.3'
 regex = "1.9.1"
 rayon = "1.7.0"
 rkyv = { version = "0.7", features = ["validation"] }
-rusqlite = { version = "0.29.0", features = ["bundled", "array", "vtab", "wasm32-wasi-vfs"] }
+rusqlite = { version = "0.32.1", features = ["bundled", "array", "vtab", "wasm32-wasi-vfs"] }
 thiserror = "1.0.40"
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }

packages/nx/src/daemon/client/client.ts

@@ -28,7 +28,7 @@ import {
   DaemonProjectGraphError,
   ProjectGraphError,
 } from '../../project-graph/error-types';
-import { IS_WASM, NxWorkspaceFiles, TaskRun } from '../../native';
+import { IS_WASM, NxWorkspaceFiles, TaskRun, TaskTarget } from '../../native';
 import { HandleGlobMessage } from '../message-types/glob';
 import {
   GET_NX_WORKSPACE_FILES,
@@ -44,7 +44,9 @@ import {
 } from '../message-types/get-files-in-directory';
 import { HASH_GLOB, HandleHashGlobMessage } from '../message-types/hash-glob';
 import {
+  GET_ESTIMATED_TASK_TIMINGS,
   GET_FLAKY_TASKS,
+  HandleGetEstimatedTaskTimings,
   HandleGetFlakyTasks,
   HandleRecordTaskRunsMessage,
   RECORD_TASK_RUNS,
@@ -357,6 +359,17 @@ export class DaemonClient {
     return this.sendToDaemonViaQueue(message);
   }
 
+  async getEstimatedTaskTimings(
+    targets: TaskTarget[]
+  ): Promise<Record<string, number>> {
+    const message: HandleGetEstimatedTaskTimings = {
+      type: GET_ESTIMATED_TASK_TIMINGS,
+      targets,
+    };
+
+    return this.sendToDaemonViaQueue(message);
+  }
+
   recordTaskRuns(taskRuns: TaskRun[]): Promise<void> {
     const message: HandleRecordTaskRunsMessage = {
       type: RECORD_TASK_RUNS,

packages/nx/src/daemon/message-types/task-history.ts

@@ -1,12 +1,24 @@
-import type { TaskRun } from '../../native';
+import type { TaskRun, TaskTarget } from '../../native';
 
 export const GET_FLAKY_TASKS = 'GET_FLAKY_TASKS' as const;
+export const GET_ESTIMATED_TASK_TIMINGS = 'GET_ESTIMATED_TASK_TIMINGS' as const;
+export const RECORD_TASK_RUNS = 'RECORD_TASK_RUNS' as const;
 
 export type HandleGetFlakyTasks = {
   type: typeof GET_FLAKY_TASKS;
   hashes: string[];
 };
 
+export type HandleGetEstimatedTaskTimings = {
+  type: typeof GET_ESTIMATED_TASK_TIMINGS;
+  targets: TaskTarget[];
+};
+
+export type HandleRecordTaskRunsMessage = {
+  type: typeof RECORD_TASK_RUNS;
+  taskRuns: TaskRun[];
+};
+
 export function isHandleGetFlakyTasksMessage(
   message: unknown
 ): message is HandleGetFlakyTasks {
@@ -18,12 +30,16 @@ export function isHandleGetFlakyTasksMessage(
   );
 }
 
-export const RECORD_TASK_RUNS = 'RECORD_TASK_RUNS' as const;
-
-export type HandleRecordTaskRunsMessage = {
-  type: typeof RECORD_TASK_RUNS;
-  taskRuns: TaskRun[];
-};
+export function isHandleGetEstimatedTaskTimings(
+  message: unknown
+): message is HandleGetEstimatedTaskTimings {
+  return (
+    typeof message === 'object' &&
+    message !== null &&
+    'type' in message &&
+    message['type'] === GET_ESTIMATED_TASK_TIMINGS
+  );
+}
 
 export function isHandleWriteTaskRunsToHistoryMessage(
   message: unknown

packages/nx/src/daemon/server/handle-task-history.ts

@@ -1,14 +1,5 @@
-import { TaskRun } from '../../native';
-import { TaskHistory } from '../../utils/task-history';
-
-let taskHistory: TaskHistory;
-
-function getTaskHistory() {
-  if (!taskHistory) {
-    taskHistory = new TaskHistory();
-  }
-  return taskHistory;
-}
+import { getTaskHistory } from '../../utils/task-history';
+import type { TaskRun, TaskTarget } from '../../native';
 
 export async function handleRecordTaskRuns(taskRuns: TaskRun[]) {
   const taskHistory = getTaskHistory();
@@ -27,3 +18,12 @@ export async function handleGetFlakyTasks(hashes: string[]) {
     description: 'handleGetFlakyTasks',
   };
 }
+
+export async function handleGetEstimatedTaskTimings(targets: TaskTarget[]) {
+  const taskHistory = getTaskHistory();
+  const history = await taskHistory.getEstimatedTaskTimings(targets);
+  return {
+    response: JSON.stringify(history),
+    description: 'handleGetEstimatedTaskTimings',
+  };
+}

packages/nx/src/daemon/server/server.ts

@@ -74,12 +74,17 @@ import { handleGetFilesInDirectory } from './handle-get-files-in-directory';
 import { HASH_GLOB, isHandleHashGlobMessage } from '../message-types/hash-glob';
 import { handleHashGlob } from './handle-hash-glob';
 import {
+  GET_ESTIMATED_TASK_TIMINGS,
+  GET_FLAKY_TASKS,
+  isHandleGetEstimatedTaskTimings,
   isHandleGetFlakyTasksMessage,
   isHandleWriteTaskRunsToHistoryMessage,
+  RECORD_TASK_RUNS,
 } from '../message-types/task-history';
 import {
   handleRecordTaskRuns,
   handleGetFlakyTasks,
+  handleGetEstimatedTaskTimings,
 } from './handle-task-history';
 import { isHandleForceShutdownMessage } from '../message-types/force-shutdown';
 import { handleForceShutdown } from './handle-force-shutdown';
@@ -240,11 +245,15 @@ async function handleMessage(socket, data: string) {
       handleHashGlob(payload.globs, payload.exclude)
     );
   } else if (isHandleGetFlakyTasksMessage(payload)) {
-    await handleResult(socket, 'GET_TASK_HISTORY_FOR_HASHES', () =>
+    await handleResult(socket, GET_FLAKY_TASKS, () =>
       handleGetFlakyTasks(payload.hashes)
     );
+  } else if (isHandleGetEstimatedTaskTimings(payload)) {
+    await handleResult(socket, GET_ESTIMATED_TASK_TIMINGS, () =>
+      handleGetEstimatedTaskTimings(payload.targets)
+    );
   } else if (isHandleWriteTaskRunsToHistoryMessage(payload)) {
-    await handleResult(socket, 'WRITE_TASK_RUNS_TO_HISTORY', () =>
+    await handleResult(socket, RECORD_TASK_RUNS, () =>
       handleRecordTaskRuns(payload.taskRuns)
     );
   } else if (isHandleForceShutdownMessage(payload)) {

packages/nx/src/generators/tree.ts

@@ -12,8 +12,6 @@ import { logger } from '../utils/logger';
 import { output } from '../utils/output';
 import { dirname, join, relative, sep } from 'path';
 import * as chalk from 'chalk';
-import { gt } from 'semver';
-import { nxVersion } from '../utils/versions';
 
 /**
  * Options to set when writing a file in the Virtual file system tree.

packages/nx/src/native/index.d.ts

@@ -41,6 +41,7 @@ export declare class NxTaskHistory {
   constructor(db: ExternalObject<Connection>)
   recordTaskRuns(taskRuns: Array<TaskRun>): void
   getFlakyTasks(hashes: Array<string>): Array<string>
+  getEstimatedTaskTimings(targets: Array<TaskTarget>): Record<string, number>
 }
 
 export declare class RustPseudoTerminal {