Separate out and fix conversation integration test (DavidSouther#92)

* Separate out and fix conversation integration test
* Remove `node` and `ailly` positionals

.github/workflows/linux.yaml

@@ -24,3 +24,5 @@ jobs:
 
       - name: Run integration tests
         run: ./integ/integ.sh
+        env:
+          AILLY_ENGINE: noop

.github/workflows/windows.yaml

@@ -25,3 +25,5 @@ jobs:
       - name: Run Integration Tests
         run: ./integ/integ.bat
         shell: cmd
+        env:
+          AILLY_ENGINE: noop

cli/args.js

@@ -1,3 +1,4 @@
+import { basename } from "node:path";
 import { parseArgs } from "node:util";
 
 export function makeArgs(argv = process.argv) {
@@ -29,12 +30,16 @@ export function makeArgs(argv = process.argv) {
       help: { type: "boolean", short: "h", default: false },
       version: { type: "boolean", default: false },
       "log-level": { type: "string", default: undefined },
+      "log-format": { type: "string", default: undefined },
       verbose: { type: "boolean", default: false, short: "v" },
-      pretty: { type: "boolean", default: false },
     },
   });
 
   // TODO assert context is content, folder, or none
+  // TODO assert log-format is pretty, json, or empty
+
+  // Remove node and ailly positionals
+  args.positionals.splice(0, args.positionals[0].match(/node(\.exe)?/) ? 2 : 1)
 
   return args;
 }
@@ -71,8 +76,8 @@ export function help() {
     --no-overwrite will not run generation on Content with an existing Response.
     --summary will show a pricing expectation before running and prompt for OK.
     -y, —-yes will skip any prompts.
-    -v, --verbose, --log-level v and verbose will set log level to info; --log-level can be a string or number and use jefri/jiffies logging levels. Ailly uses warn for reporting details on errors, info for general runtime progress, and debug for details of requests and responses. Logs are structures in JSONL.
-    --log-pretty Use a pretty log formatter. Default is a structured JSON log formatter, unless using --prompt or in a pipe, when using pretty.
+    -v, --verbose, --log-level v and verbose will set log level to info; --log-level can be a string or number and use jefri/jiffies logging levels. Ailly uses warn for reporting details on errors, info for general runtime progress, and debug for details of requests and responses.
+    --log-format json or pretty; default is pretty when run in a pipe. JSON prints in JSONL format.
 
     --version will print the cli and core versions
     -h, --help will print this message and exit.

cli/fs.js

@@ -41,14 +41,15 @@ export async function loadFs(fs, args) {
     requestLimit: args.values['request-limit'] ?? args.values.model?.includes("opus") ? 1 : undefined,
   });
 
-  const positionals = args.positionals.slice(2).map(a => resolve(join(root, a)));
+  const positionals = args.positionals.map(a => resolve(join(root, a)));
   const hasPositionals = positionals.length > 0;
   const hasPrompt = args.values.prompt !== undefined && args.values.prompt !== "";
   const isPipe = !hasPositionals && hasPrompt;
   const logLevel = args.values['log-level'] ?? (args.values.verbose ? 'verbose' : (isPipe ? 'silent' : undefined));
+  const logFormat = args.values['log-format'] ?? (isPipe ? "pretty" : "json");
   ailly.Ailly.LOGGER.console = LOGGER.console = isPipe ? new Console(process.stderr, process.stderr) : global.console;
   ailly.Ailly.LOGGER.level = LOGGER.level = getLogLevel(logLevel);
-  if (args.values.pretty || isPipe) LOGGER.format = ailly.Ailly.LOGGER.format = basicLogFormatter;
+  ailly.Ailly.LOGGER.format = LOGGER.format = logFormat == "json" ? JSON.stringify : basicLogFormatter;
 
   const system = args.values.system ?? "";
 

cli/package.json

@@ -27,7 +27,7 @@
   },
   "dependencies": {
     "@ailly/core": "1.3.1",
-    "@davidsouther/jiffies": "^2.2.2",
+    "@davidsouther/jiffies": "^2.2.3",
     "yaml": "^2.4.1"
   },
   "devDependencies": {

core/package.json

@@ -27,7 +27,7 @@
   "homepage": "https://github.com/DavidSouther/ailly#readme",
   "dependencies": {
     "@aws-sdk/client-bedrock-runtime": "^3.427.0",
-    "@davidsouther/jiffies": "^2.2.2",
+    "@davidsouther/jiffies": "^2.2.3",
     "@dqbd/tiktoken": "^1.0.7",
     "gitignore-parser": "^0.0.2",
     "gray-matter": "^4.0.3",

integ/05_conversation/conversation.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -x
+set -e
+
+cd $(dirname $0)
+
+rm -f ./out ./err
+
+###
+
+echo "basic conversation"
+npx ailly --root ./root --log-format pretty --prompt "This is a conversation with system and two files." > >(tee ./out) 2> >(tee ./err >&2)
+tail ./out ./err
+grep -vq '"name":"@ailly/core"' ./out # No JSONL output in `out`
+[ ! -s ./err ]                        # No error output at all
+
+MESSAGES=(
+  "You are running an integration test."
+  "user: File a."
+  "user: File b."
+  "This is a conversation with system and two files."
+)
+for M in "${MESSAGES[@]}"; do
+  grep -q "$M" ./out
+done
+
+echo "(all conversation messages checked)"
+rm -f ./out ./err
+
+###
+
+echo "verbose conversation"
+npx ailly --root ./root --log-format pretty --verbose --prompt "This is a conversation with system and two files." > >(tee ./out) 2> >(tee ./err >&2)
+tail ./out ./err
+
+MESSAGES=(
+  "You are running an integration test."
+  "user: File a."
+  "user: File b."
+  "This is a conversation with system and two files."
+)
+for M in "${MESSAGES[@]}"; do
+  grep -q "$M" ./out
+done
+MESSAGES=(
+  "Found 2 at or below"
+  "Ready to generate 1 messages"
+  "Preparing /dev/stdout"
+  "All 1 requests finished"
+)
+for M in "${MESSAGES[@]}"; do
+  grep -vq "$M" ./err
+done
+
+echo "(all verbose conversation messages checked)"
+rm -f ./out ./err

integ/10_std_pipes/pipes.sh

@@ -19,3 +19,4 @@ rm out.txt
 [ -f out.txt ]
 grep -vq '{"name":"@ailly/core"' out.txt
 grep -q "Explain this code" out.txt
+rm out.txt

integ/integ.sh

@@ -27,52 +27,8 @@ grep -q 'Edited' 04_edit/file.txt
 git restore 04_edit/file.txt
 unset AILLY_NOOP_RESPONSE
 
-echo "conversation"
-npx ailly --root 05_conversation --prompt "This is a conversation with system and two files." > >(tee 05_conversation/out) 2> >(tee 05_conversation/err >&2)
-cat 05_conversation/out
-cat 05_conversation/err
-MESSAGES=(
-  "Found 2 at or below"
-  "Ready to generate 1 messages"
-  "Preparing /dev/stdout"
-  "All 1 requests finished"
-)
-for M in "${MESSAGES[@]}"; do
-  grep -vq "$M" 05_conversation/out
-done
-
-MESSAGES=(
-  "You are running an integration test."
-  "user: File a."
-  "user: File b."
-  "This is a conversation with system and two files."
-)
-for M in "${MESSAGES[@]}"; do
-  grep -q "$M" 05_conversation/out
-done
-echo "(all conversation messages checked)"
-rm 05_conversation/out 05_conversation/err
-
-echo "verbose conversation"
-npx ailly --root 05_conversation --prompt "This is a conversation with system and two files." --verbose > >(tee 05_conversation/out) 2> >(tee 05_conversation/err >&2)
-cat 05_conversation/out
-cat 05_conversation/err
-[ ! -s 05_conversation/err ]
-MESSAGES=(
-  "Found 2 at or below"
-  "Ready to generate 1 messages"
-  "Preparing /dev/stdout"
-  "All 1 requests finished"
-  "You are running an integration test."
-  "user: File a."
-  "user: File b."
-  "This is a conversation with system and two files."
-)
-for M in "${MESSAGES[@]}"; do
-  grep -q "$M" 05_conversation/out
-done
-rm 05_conversation/out 05_conversation/err
-echo "(all verbose conversation messages checked)"
+echo "conversations"
+./05_conversation/conversation.sh
 
 echo "Pipes"
 ./10_std_pipes/pipes.sh

web/package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@ailly/core": "^1.3.0",
-    "@davidsouther/jiffies": "^2.2.0",
+    "@davidsouther/jiffies": "^2.2.3",
     "marked-react": "^2.0.0",
     "next": "^14.2.2",
     "react": "^18",