Integration tests for --max-depth (DavidSouther#89)

cli/args.js

@@ -21,6 +21,7 @@ export function makeArgs(argv = process.argv) {
       prompt: { type: "string", default: process.env["AILLY_PROMPT"], short: "p" },
       system: { type: "string", default: process.env["AILLY_SYSTEM"], short: "s" },
       "request-limit": { type: "string", default: process.env["AILLY_REQUEST_LIMIT"] },
+      "max-depth": { type: "string", default: "1" },
       temperature: { type: "string", default: "" },
       "update-db": { type: "boolean", default: false },
       "query-db": { type: "string", default: "" },
@@ -72,6 +73,7 @@ export function help() {
     --plugin can load a custom RAG plugin. Specify a path to import with "file://./path/to/plugin.mjs". plugin.mjs must export a single default function that meets the PluginBuilder interface in core/src/plugin/index.ts
     --template-view loads a YAML or JSON file to use as a view for the prompt templates. This view will be merged after global, engine, and plugin views but before system and template views.
     --request-limit will limit the number of requests per call to the provided value. Default value is 5, except for Opus with a default of 1.
+    --max-depth will allow loading content below the current root. Default 1, or only the root folder. 0 or negative numbers will load no content.
 
     --no-overwrite will not run generation on Content with an existing Response.
     --summary will show a pricing expectation before running and prompt for OK.

cli/fs.js

@@ -52,11 +52,13 @@ export async function loadFs(fs, args) {
   ailly.Ailly.LOGGER.format = LOGGER.format = logFormat == "json" ? JSON.stringify : basicLogFormatter;
 
   const system = args.values.system ?? "";
+  const depth = Number(args.values["max-depth"]);
 
   let context = await ailly.content.load(
     fs,
     system ? [{ content: system, view: {} }] : [],
-    settings
+    settings,
+    depth
   );
 
   let content = /* @type {string[]} */[];

core/src/content/content.ts

@@ -242,16 +242,19 @@ export async function loadAillyRc(
  * 3. If it is a folder
  *    1. Find all files that are not denied by .gitignore
  *    2. Apply the above logic.
- * @param fs the file system abstraction
- * @param system the system message chain
- * @param meta current head matter & settings
+ * @param fs the file system abstraction.
+ * @param system the system message chain.
+ * @param meta current head matter & settings.
+ * @param depth maximum depth to load. `1` loads only the cwd; 0 or negative loads no content.
  * @returns
  */
 export async function loadContent(
   fs: FileSystem,
   system: System[] = [],
-  meta: ContentMeta = {}
+  meta: ContentMeta = {},
+  depth: number = Number.MAX_SAFE_INTEGER
 ): Promise<Record<string, Content>> {
+  if (depth < 1) return {};
   LOGGER.debug(`Loading content from ${fs.cwd()}`);
   [system, meta] = await loadAillyRc(fs, system, meta);
   if (meta.skip) {
@@ -293,7 +296,12 @@ export async function loadContent(
     for (const folder of dir.folders) {
       if (folder.name == ".vectors") continue;
       fs.pushd(folder.name);
-      let contents = await loadContent(fs, system, meta);
+      let contents = await loadContent(
+        fs,
+        system,
+        meta,
+        depth ? depth - 1 : undefined
+      );
       Object.assign(folders, contents);
       fs.popd();
     }

core/src/engine/bedrock/bedrock.test.ts

@@ -51,7 +51,7 @@ describe("bedrock claude3", () => {
           "/root/b": "prompt b",
         })
       );
-      const context = await loadContent(fs, [], settings);
+      const context = await loadContent(fs, [], settings, 2);
       return { root, settings, context };
     }, beforeEach);
 

extension/src/generate.ts

@@ -31,7 +31,7 @@ export async function generate(
   });
 
   // Load content
-  const context = await ailly.content.load(fs);
+  const context = await ailly.content.load(fs, [], {}, 1);
   const content = Object.values(context).filter((c) => c.path.startsWith(path));
   if (content.length == 0) return;
   if (edit) {

integ/11_max_depth/max_depth.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+set -x
+
+cd $(dirname $0)
+
+ailly --root ./root --log-level debug >log
+grep -q "Ready to generate 1 messages" log
+
+ailly --root ./root --log-level debug --max-depth 2 >log
+grep -q "Ready to generate 2 messages" log
+
+rm log

integ/11_max_depth/root/deep/deep.txt

@@ -0,0 +1 @@
+deep

integ/11_max_depth/root/root.txt

@@ -0,0 +1 @@
+root

integ/integ.sh

@@ -32,3 +32,6 @@ echo "conversations"
 
 echo "Pipes"
 ./10_std_pipes/pipes.sh
+
+echo "Max depth"
+./11_max_depth/max_depth.sh