add --system-compose-file option (livecycle#93)

- remove config file for now
- add new option to allow specifying the compose file path without overriding the default loading order

README.md

@@ -140,13 +140,15 @@ When using the default `*.livecycle.run` domain, environments are publicly acces
 
 ## Configuration files
 
-For most purposes, Preevy extracts its runtime settings from the [Compose file](https://docs.docker.com/compose/compose-file/03-compose-file/), and no additional configuration is required.
+Preevy extracts its runtime settings from the [Compose file](https://docs.docker.com/compose/compose-file/03-compose-file/).
 
-The Compose file is loaded using the `docker compose` command and thus follows the same [rules](https://docs.docker.com/compose/reference/#use--f-to-specify-name-and-path-of-one-or-more-compose-files) regarding default loading order. Just like with `docker compose`, you can use the `--file | -f` option with most of the commands to specify path(s) for the Compose file.
+Just like with `docker compose`, you can use the global `--file | -f` option to specify path(s) for the Compose file. If not specified, the [default loading order](https://docs.docker.com/compose/reference/#use--f-to-specify-name-and-path-of-one-or-more-compose-files) is used. Multiple files are [supported](https://docs.docker.com/compose/extends/#multiple-compose-files) just like with `docker compose`.
+
+An additional option `--system-compose-file` can be used to specify paths to Compose files without overriding the default loading order. This is useful for scripts invoking the Preevy CLI (e.g, a GitHub Action), to accept user-provided compose files (including the default loading order) while ensuring a specific file is always loaded.
 
 ### Preevy-specific configuration
 
-Additional configuration, if needed, can be specified by adding a `x-preevy` top-level element to the Compose file(s). Currently only the `plugins` section is supported:
+Additional Preevy-specific configuration, if needed, can be specified by adding a `x-preevy` top-level element to the Compose file(s). Currently only the `plugins` section is supported:
 
 ```yaml
 services:
@@ -160,15 +162,6 @@ x-preevy:
 See [Plugins](#plugins) below.
 <!--lint enable double-link-->
 
-In addition to the Compose file, a preevy-specific configuration file can be specified by the `--config | -c` option. The file can be in YAML or JSON format, and its schema corresponds to the `x-preevy` top-level element:
-
-```yaml
-plugins:
-  ...
-```
-
-If the `--config | -c` option is not specified, Preevy attempts to load `preevy.yaml`, `preevy.yml` and `preevy.json`, in this order, from the current working directory.
-
 ## Plugins
 
 Plugins are a way to extend Preevy's functionality via externally-published NPM packages.

packages/cli-common/src/commands/base-command.ts

@@ -4,7 +4,7 @@ import {
 } from '@preevy/core'
 import { asyncReduce } from 'iter-tools-es'
 import { commandLogger } from '../lib/log'
-import { configFlags } from '../lib/flags'
+import { composeFlags } from '../lib/flags'
 
 // eslint-disable-next-line no-use-before-define
 export type Flags<T extends typeof Command> = Interfaces.InferredFlags<typeof BaseCommand['baseFlags'] & T['flags']>
@@ -29,7 +29,7 @@ abstract class BaseCommand<T extends typeof Command=typeof Command> extends Comm
         { type: 'none', flags: ['log-level'] },
       ],
     }),
-    ...configFlags,
+    ...composeFlags,
   }
 
   protected flags!: Flags<T>

packages/cli-common/src/hooks/init/load-plugins.ts

@@ -1,9 +1,8 @@
 import { Hook as OclifHook, Command } from '@oclif/core'
 import { Parser } from '@oclif/core/lib/parser/parse'
 import { Config, Topic } from '@oclif/core/lib/interfaces'
-import { localComposeClient, config as coreConfig, ComposeModel } from '@preevy/core'
-import loadConfig = coreConfig.loadConfig
-import { composeFlags, configFlags } from '../../lib/flags'
+import { localComposeClient, ComposeModel, resolveComposeFiles } from '@preevy/core'
+import { composeFlags } from '../../lib/flags'
 import { addPluginFlags, loadPlugins, hooksFromPlugins, addPluginCommands } from '../../lib/plugins'
 
 type InternalConfig = Config & {
@@ -12,20 +11,20 @@ type InternalConfig = Config & {
 
 export const initHook: OclifHook<'init'> = async function hook({ config, id: _id, argv }) {
   const { flags } = await new Parser({
-    flags: { ...composeFlags, ...configFlags },
+    flags: { ...composeFlags },
     strict: false,
     args: {},
     context: undefined,
     argv,
   }).parse()
 
   const userModelOrError = await localComposeClient({
-    composeFiles: flags.file,
+    composeFiles: resolveComposeFiles({ userSpecifiedFiles: flags.file, systemFiles: flags['system-compose-file'] }),
     projectName: flags.project,
   }).getModelOrError()
 
   const userModel = userModelOrError instanceof Error ? {} as ComposeModel : userModelOrError
-  const preevyConfig = await loadConfig(flags.config || [], userModel)
+  const preevyConfig = userModel['x-preevy'] ?? {}
   const loadedPlugins = await loadPlugins(preevyConfig, { userModel, oclifConfig: config, argv })
   const commands = addPluginFlags(addPluginCommands(config.commands, loadedPlugins), loadedPlugins);
 

packages/cli-common/src/index.ts

@@ -1,6 +1,6 @@
 export * from './lib/plugins/model'
 export { HookName, HookFunc, HooksListeners, Hooks } from './lib/hooks'
 export { PluginContext, PluginInitContext } from './lib/plugins/context'
-export { composeFlags, envIdFlags, configFlags } from './lib/flags'
+export { composeFlags, envIdFlags } from './lib/flags'
 export { initHook } from './hooks/init/load-plugins'
 export { default as BaseCommand } from './commands/base-command'

packages/cli-common/src/lib/flags.ts

@@ -7,23 +7,22 @@ export const composeFlags = {
     required: false,
     char: 'f',
     default: [],
+    helpGroup: 'GLOBAL',
+  }),
+  'system-compose-file': Flags.string({
+    description: 'Add extra Compose configuration file without overriding the defaults',
+    multiple: true,
+    required: false,
+    default: [],
+    helpGroup: 'GLOBAL',
   }),
   project: Flags.string({
     char: 'p',
     description: 'Project name. Defaults to the Compose project name',
     required: false,
-  }),
-} as const
-
-export const configFlags = {
-  config: Flags.string({
-    description: 'Path to a JSON/YAML configuration file containing the Preevy config',
-    multiple: true,
-    required: false,
-    char: 'c',
     helpGroup: 'GLOBAL',
   }),
-}
+} as const
 
 export const envIdFlags = {
   id: Flags.string({

packages/cli/src/commands/down.ts

@@ -1,15 +1,14 @@
 import { Flags } from '@oclif/core'
 import { findAmbientEnvId, withSpinner } from '@preevy/core'
 import DriverCommand from '../driver-command'
-import { envIdFlags, composeFlags } from '../common-flags'
+import { envIdFlags } from '../common-flags'
 
 // eslint-disable-next-line no-use-before-define
 export default class Down extends DriverCommand<typeof Down> {
   static description = 'Delete preview environments'
 
   static flags = {
     ...envIdFlags,
-    ...composeFlags,
     force: Flags.boolean({
       description: 'Do not error if the environment is not found',
       default: false,

packages/cli/src/commands/logs.ts

@@ -6,15 +6,14 @@ import {
   findAmbientEnvId, localComposeClient, wrapWithDockerSocket,
 } from '@preevy/core'
 import DriverCommand from '../driver-command'
-import { envIdFlags, composeFlags } from '../common-flags'
+import { envIdFlags } from '../common-flags'
 
 // eslint-disable-next-line no-use-before-define
 export default class Logs extends DriverCommand<typeof Logs> {
   static description = 'Show logs for an existing environment'
 
   static flags = {
     ...envIdFlags,
-    ...composeFlags,
     ...ux.table.flags(),
   }
 
@@ -46,7 +45,7 @@ export default class Logs extends DriverCommand<typeof Logs> {
     const envId = flags.id || await findAmbientEnvId(projectName)
     log.debug(`envId: ${envId}`)
 
-    const model = await localComposeClient({ composeFiles: flags.file, projectName }).getModel()
+    const model = await this.ensureUserModel()
 
     // exclude docker proxy service unless explicitly specified
     const modelServices = Object.keys(model.services ?? {})

packages/cli/src/commands/up.ts

@@ -8,7 +8,7 @@ import {
 import { asyncReduce } from 'iter-tools-es'
 import MachineCreationDriverCommand from '../machine-creation-driver-command'
 import { carefulBooleanPrompt } from '../prompt'
-import { envIdFlags, composeFlags } from '../common-flags'
+import { envIdFlags } from '../common-flags'
 
 const confirmHostFingerprint = async (
   { hostKeyFingerprint: hostKeySignature, hostname, port }: Parameters<HostKeySignatureConfirmer>[0],
@@ -28,7 +28,6 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
 
   static flags = {
     ...envIdFlags,
-    ...composeFlags,
     'tunnel-url': Flags.string({
       description: 'Tunnel url, specify ssh://hostname[:port] or ssh+tls://hostname[:port]',
       char: 't',
@@ -112,8 +111,9 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       userModel,
       userSpecifiedProjectName: flags.project,
       userSpecifiedEnvId: flags.id,
+      userSpecifiedComposeFiles: flags.file,
+      systemComposeFiles: flags['system-compose-file'],
       tunnelOpts,
-      composeFiles: flags.file,
       log: this.logger,
       dataDir: this.config.dataDir,
       sshKey: keyPair,

packages/cli/src/commands/urls.ts

@@ -1,15 +1,14 @@
 import { Args, ux } from '@oclif/core'
 import { commands, findAmbientEnvId } from '@preevy/core'
 import DriverCommand from '../driver-command'
-import { envIdFlags, composeFlags } from '../common-flags'
+import { envIdFlags } from '../common-flags'
 
 // eslint-disable-next-line no-use-before-define
 export default class Urls extends DriverCommand<typeof Urls> {
   static description = 'Show urls for an existing environment'
 
   static flags = {
     ...envIdFlags,
-    ...composeFlags,
     ...ux.table.flags(),
   }
 

packages/core/src/commands/up/index.ts

@@ -15,6 +15,7 @@ import { Machine, MachineCreationDriver, MachineDriver } from '../../driver'
 import { REMOTE_DIR_BASE, remoteProjectDir } from '../../remote-files'
 import { Logger } from '../../log'
 import { Tunnel, tunnelUrl } from '../../tunneling'
+import { resolveComposeFiles } from '../../compose'
 
 const createCopiedFileInDataDir = (
   { projectLocalDataDir, filesToCopy, remoteDir } : {
@@ -56,8 +57,9 @@ const up = async ({
   userSpecifiedProjectName,
   userSpecifiedEnvId,
   userSpecifiedServices,
+  userSpecifiedComposeFiles,
+  systemComposeFiles,
   log,
-  composeFiles: userComposeFiles,
   dataDir,
   sshKey,
   allowedSshHostKeys: hostKey,
@@ -73,15 +75,14 @@ const up = async ({
   userSpecifiedProjectName: string | undefined
   userSpecifiedEnvId: string | undefined
   userSpecifiedServices: string[]
+  userSpecifiedComposeFiles: string[]
+  systemComposeFiles: string[]
   log: Logger
-  composeFiles: string[]
   dataDir: string
   sshKey: SSHKeyConfig
   sshTunnelPrivateKey: string
   allowedSshHostKeys: Buffer
 }): Promise<{ machine: Machine; tunnels: Tunnel[]; envId: string }> => {
-  log.debug('Normalizing compose files')
-
   const projectName = userSpecifiedProjectName ?? userModel.name
   const remoteDir = remoteProjectDir(projectName)
 
@@ -101,10 +102,17 @@ const up = async ({
     }),
   }), {})
 
+  const composeFiles = resolveComposeFiles({
+    userSpecifiedFiles: userSpecifiedComposeFiles,
+    systemFiles: systemComposeFiles,
+  })
+
+  log.debug(`Using compose files: ${composeFiles.join(', ')}`)
+
   // Now that we have the generated variables, we can create a new client and inject
   // them into it, to create the actual compose configurations
   const composeClientWithInjectedArgs = localComposeClient(
-    { composeFiles: userComposeFiles, env: composeEnv, projectName: userSpecifiedProjectName }
+    { composeFiles, env: composeEnv, projectName: userSpecifiedProjectName }
   )
 
   const { model: fixedModel, filesToCopy } = await fixModelForRemote(

packages/core/src/compose/client.ts

@@ -84,11 +84,13 @@ export const localComposeClient = (
     return [null, null, null]
   }
 
+  const fileArgs = composeFileArgs(composeFiles, projectName)
+
   const spawnComposeArgs = (...[args, opts]: ParametersExceptFirst<typeof spawn>): Parameters<typeof spawn> => [
     'docker',
     [
       'compose',
-      ...composeFileArgs(composeFiles, projectName),
+      ...fileArgs,
       ...args,
     ],
     {

packages/core/src/compose/files.ts

@@ -0,0 +1,39 @@
+import fs from 'fs'
+
+export type ComposeFiles = {
+  userSpecifiedFiles: string[]
+  systemFiles: string[]
+}
+
+const DEFAULT_BASE_FILES = ['compose', 'docker-compose']
+const DEFAULT_OVERRIDE_FILES = DEFAULT_BASE_FILES.map(f => `${f}.override`)
+const YAML_EXTENSIONS = ['yaml', 'yml']
+
+const oneYamlFile = (baseNames: string[], type: string) => {
+  const existingFiles = baseNames
+    .flatMap(f => YAML_EXTENSIONS.map(e => `${f}.${e}`))
+    .filter(f => fs.existsSync(f))
+
+  if (!existingFiles.length) {
+    return undefined
+  }
+
+  if (existingFiles.length > 1) {
+    throw new Error(`Multiple ${type} files found: ${existingFiles.join(', ')}`)
+  }
+
+  return existingFiles[0]
+}
+
+const findDefaultFiles = () => {
+  const defaultFile = oneYamlFile(DEFAULT_BASE_FILES, 'default Compose')
+  if (!defaultFile) {
+    return []
+  }
+  const overrideFile = oneYamlFile(DEFAULT_OVERRIDE_FILES, 'default Compose override')
+  return overrideFile ? [defaultFile, overrideFile] : [defaultFile]
+}
+
+export const resolveComposeFiles = (
+  { userSpecifiedFiles, systemFiles }: ComposeFiles,
+): string[] => [...userSpecifiedFiles.length ? userSpecifiedFiles : findDefaultFiles(), ...systemFiles]

packages/core/src/compose/index.ts

@@ -1,2 +1,3 @@
 export * from './client'
 export * from './model'
+export { ComposeFiles, resolveComposeFiles } from './files'

packages/core/src/index.ts

@@ -4,7 +4,7 @@ export { Machine, PartialMachine, MachineDriver, MachineCreationDriver, MachineC
 export { profileStore, Profile, ProfileStore } from './profile'
 export { telemetryEmitter, registerEmitter, wireProcessExit, createTelemetryEmitter } from './telemetry'
 export { fsTypeFromUrl, Store, VirtualFS, localFsFromUrl } from './store'
-export { localComposeClient, ComposeModel } from './compose'
+export { localComposeClient, ComposeModel, resolveComposeFiles } from './compose'
 export { withSpinner } from './spinner'
 export { findAmbientEnvId } from './env-id'
 export { sshKeysStore } from './state'

packages/plugin-github-pr-link/src/commands/link-github-pr.ts

@@ -1,5 +1,5 @@
-import { Command, Interfaces } from '@oclif/core'
-import { BaseCommand, composeFlags, envIdFlags } from '@preevy/cli-common'
+import { Command, Flags, Interfaces } from '@oclif/core'
+import { BaseCommand, envIdFlags } from '@preevy/cli-common'
 import { Octokit } from 'octokit'
 import { findAmbientEnvId, FlatTunnel } from '@preevy/core'
 import { flagsDef, ParsedFlags } from '../flags'
@@ -16,7 +16,6 @@ class LinkGithubPr extends BaseCommand<typeof LinkGithubPr> {
   static description = 'Link a GitHub Pull Request to an existing environment'
 
   static flags = {
-    ...composeFlags,
     ...envIdFlags,
     ...flagsDef,
   }