Skip to content

Commit

Permalink
Switch the template build to local (e2b-dev#338)
Browse files Browse the repository at this point in the history 
Change CLI to build the docker image locally
  • Loading branch information
@jakubno
jakubno authored Mar 24, 2024
1 parent bd5874f commit 2bafaf0
Show file tree
Hide file tree
Showing 33 changed files with 337 additions and 2,176 deletions.
7 changes: 7 additions & 0 deletions .changeset/late-rice-hammer.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"e2b": minor
"@e2b/cli": minor
"@e2b/python-sdk": patch
---

Change template build to local docker
7 changes: 2 additions & 5 deletions packages/cli/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@
"update-deps": "ncu -u && pnpm i"
},
"devDependencies": {
"@types/command-exists": "^1.2.3",
"@types/inquirer": "^9.0.7",
"@types/node": "^18.18.6",
"@types/tar-fs": "^2.0.3",
"@types/update-notifier": "6.0.5",
"knip": "^2.33.4",
"npm-check-updates": "^16.14.6",
Expand All @@ -48,21 +48,18 @@
"e2b": "dist/index.js"
},
"dependencies": {
"@balena/dockerignore": "^1.0.2",
"@iarna/toml": "^2.2.5",
"@nodelib/fs.walk": "^2.0.0",
"async-listen": "^3.0.1",
"boxen": "^7.1.1",
"chalk": "^5.3.0",
"cli-highlight": "^2.1.11",
"command-exists": "^1.2.9",
"commander": "^11.1.0",
"console-table-printer": "^2.11.2",
"e2b": "^0.12.6",
"ignore": "^5.2.4",
"inquirer": "^9.2.12",
"open": "^9.1.0",
"strip-ansi": "^7.1.0",
"tar-fs": "^3.0.4",
"update-notifier": "5.1.0",
"yup": "^1.3.2"
},
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/commands/sandbox/list.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ export const listCommand = new commander.Command('list')
{ name: 'templateID', alignment: 'left', title: 'Template ID' },
{ name: 'alias', alignment: 'left', title: 'Alias' },
{ name: 'startedAt', alignment: 'left', title: 'Started at' },
{ name: 'cpuCount', alignment: 'left', title: 'CPUs' },
{ name: 'memoryMB', alignment: 'left', title: 'RAM MB' },
{ name: 'cpuCount', alignment: 'left', title: 'vCPUs' },
{ name: 'memoryMB', alignment: 'left', title: 'RAM MiB' },
{ name: 'metadata', alignment: 'left', title: 'Metadata' },
],
disabledColumns: ['clientID'],
Expand Down
208 changes: 120 additions & 88 deletions packages/cli/src/commands/template/build.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import * as path from 'path'
import * as e2b from 'e2b'
import * as stripAnsi from 'strip-ansi'
import * as boxen from 'boxen'

import commandExists from 'command-exists'
import { wait } from 'src/utils/wait'
import { ensureAccessToken } from 'src/api'
import { getFiles, getRoot } from 'src/utils/filesystem'
import { getRoot } from 'src/utils/filesystem'
import {
asBold,
asBuildLogs,
Expand All @@ -20,15 +20,26 @@ import {
withDelimiter,
} from 'src/utils/format'
import { pathOption } from 'src/options'
import { createBlobFromFiles } from 'src/docker/archive'
import { defaultDockerfileName, fallbackDockerfileName } from 'src/docker/constants'
import { configName, getConfigPath, loadConfig, maxContentSize, saveConfig } from 'src/config'
import { estimateContentLength } from '../../utils/form'
import { configName, getConfigPath, loadConfig, saveConfig } from 'src/config'
import * as child_process from 'child_process'

const templateCheckInterval = 1_000 // 1 sec
const templateCheckInterval = 500 // 0.5 sec

const getTemplate = e2b.withAccessToken(
e2b.api.path('/templates/{templateID}/builds/{buildID}').method('get').create(),
e2b.api.path('/templates/{templateID}/builds/{buildID}/status').method('get').create(),
)

const requestTemplateBuild = e2b.withAccessToken(
e2b.api.path('/templates').method('post').create(),
)

const requestTemplateRebuild = e2b.withAccessToken(
e2b.api.path('/templates/{templateID}').method('post').create(),
)

const triggerTemplateBuild = e2b.withAccessToken(
e2b.api.path('/templates/{templateID}/builds/{buildID}').method('post').create(),
)

export const buildCommand = new commander.Command('build')
Expand Down Expand Up @@ -67,10 +78,12 @@ export const buildCommand = new commander.Command('build')
.option(
'--cpu-count <cpu-count>',
'specify the number of CPUs that will be used to run the sandbox. The default value is 2.',
parseInt,
)
.option(
'--memory-mb <memory-mb>',
'specify the amount of memory in megabytes that will be used to run the sandbox. Must be an even number. The default value is 512.',
parseInt,
)
.alias('bd')
.action(
Expand All @@ -81,11 +94,19 @@ export const buildCommand = new commander.Command('build')
dockerfile?: string;
name?: string;
cmd?: string;
cpuCount?: string;
memoryMb?: string;
cpuCount?: number;
memoryMb?: number;
},
) => {
try {
const dockerInstalled = commandExists.sync('docker')
if (!dockerInstalled) {
console.error(
'Docker is required to build and push the sandbox template. Please install Docker and try again.',
)
process.exit(1)
}

const accessToken = ensureAccessToken()
process.stdout.write('\n')

Expand All @@ -99,7 +120,8 @@ export const buildCommand = new commander.Command('build')

let dockerfile = opts.dockerfile
let startCmd = opts.cmd

let cpuCount= opts.cpuCount
let memoryMB = opts.memoryMb

const root = getRoot(opts.path)
const configPath = getConfigPath(root)
Expand All @@ -123,6 +145,8 @@ export const buildCommand = new commander.Command('build')
templateID = config.template_id
dockerfile = opts.dockerfile || config.dockerfile
startCmd = opts.cmd || config.start_cmd
cpuCount = opts.cpuCount || config.cpu_count
memoryMB = opts.memoryMb || config.memory_mb
}

if (config && templateID && config.template_id !== templateID) {
Expand All @@ -131,23 +155,13 @@ export const buildCommand = new commander.Command('build')
process.exit(1)
}

const filePaths = await getFiles(root, {
respectGitignore: false,
respectDockerignore: true,
overrideIgnoreFor: [dockerfile || defaultDockerfileName],
})

if (newName && config?.template_name && newName !== config?.template_name) {
console.log(
`The sandbox template name will be changed from ${asLocal(config.template_name)} to ${asLocal(newName)}.`,
)
}
const name = newName || config?.template_name

console.log(
`Preparing sandbox template building (${filePaths.length} files in Docker build context). ${!filePaths.length ? `If you are using ${asLocal('.dockerignore')} check if it is configured correctly.` : ''}`,
)

const { dockerfileContent, dockerfileRelativePath } = getDockerfile(
root,
dockerfile,
Expand All @@ -159,79 +173,71 @@ export const buildCommand = new commander.Command('build')
)} that will be used to build the sandbox template.`,
)

const body = new FormData()

body.append('dockerfile', dockerfileContent)

// It should be possible to pipe directly to the API
// instead of creating a blob in memory then streaming.
const blob = await createBlobFromFiles(
root,
filePaths,
dockerfileRelativePath !== fallbackDockerfileName
? [
{
oldPath: dockerfileRelativePath,
newPath: fallbackDockerfileName,
},
]
: [],
maxContentSize,
)
body.append('buildContext', blob, 'env.tar.gz.e2b')

if (name) {
body.append('alias', name)
}

if (startCmd) {
body.append('startCmd', startCmd)
}

if (opts.cpuCount) {
body.append('cpuCount', opts.cpuCount)
const body = {
alias: name,
startCmd: startCmd,
cpuCount: cpuCount,
memoryMB: memoryMB,
dockerfile: dockerfileContent,
}

if (opts.memoryMb) {
if (parseInt(opts.memoryMb) % 2 !== 0) {
if (opts.memoryMb % 2 !== 0) {
console.error(
`The memory in megabytes must be an even number. You provided ${asLocal(opts.memoryMb)}.`,
`The memory in megabytes must be an even number. You provided ${asLocal(opts.memoryMb.toFixed(0))}.`,
)
process.exit(1)
}

body.append('memoryMB', opts.memoryMb)
}

const estimatedSize = estimateContentLength(body)
if (estimatedSize > maxContentSize) {
console.error(
`The sandbox template build context is too large ${asLocal(`${Math.round(estimatedSize / 1024 / 1024 * 100) / 100} MiB`)}. The maximum size is ${asLocal(
`${maxContentSize / 1024 / 1024} MiB.`)}\n\nCheck if you are not including unnecessary files in the build context (e.g. node_modules)`,
)
process.exit(1)
}

const build = await buildTemplate(accessToken, body, !!config, configPath, templateID)
const template = await requestBuildTemplate(accessToken, body, !!config, relativeConfigPath, templateID)
templateID = template.templateID

console.log(
`Started building the sandbox template ${asFormattedSandboxTemplate(
build,
`Requested build for the sandbox template ${asFormattedSandboxTemplate(
template,
)} `,
)

await waitForBuildFinish(accessToken, build.templateID, build.buildID, name)

await saveConfig(
configPath,
{
template_id: build.templateID,
template_id: template.templateID,
dockerfile: dockerfileRelativePath,
template_name: name,
start_cmd: startCmd,
cpu_count: cpuCount,
memory_mb: memoryMB,
},
true,
)


child_process.execSync(`echo ${accessToken} | docker login docker.e2b-staging.com -u _e2b_access_token --password-stdin`, { stdio: 'inherit', cwd: root})
process.stdout.write('\n')

console.log('Building docker image...')
child_process.execSync(`DOCKER_CLI_HINTS=false docker build . -f ${dockerfileRelativePath} --platform linux/amd64 -t docker.e2b-staging.com/e2b-dev/e2b-custom-environments/${templateID}:${template.buildID}`, { stdio: 'inherit', cwd: root})

process.stdout.write('\n')
console.log('Pushing docker image...')
child_process.execSync(`docker push docker.e2b-staging.com/e2b-dev/e2b-custom-environments/${templateID}:${template.buildID}`, { stdio: 'inherit', cwd: root })

process.stdout.write('\n')

console.log('Triggering build...')
await triggerBuild(accessToken, templateID, template.buildID)

console.log(
`Triggered build for the sandbox template ${asFormattedSandboxTemplate(
template,
)} `,
)

console.log('Waiting for build to finish...')
await waitForBuildFinish(accessToken, templateID, template.buildID, name)

process.exit(0)
} catch (err: any) {
console.error(err)
process.exit(1)
Expand Down Expand Up @@ -424,9 +430,9 @@ function getDockerfile(root: string, file?: string) {
)
}

async function buildTemplate(
async function requestBuildTemplate(
accessToken: string,
body: FormData,
args: e2b.paths['/templates']['post']['requestBody']['content']['application/json'],
hasConfig: boolean,
configPath: string,
templateID?: string,
Expand All @@ -436,27 +442,18 @@ async function buildTemplate(
'logs'
>
> {
const res = await fetch(e2b.API_HOST + (templateID ? `/templates/${templateID}` : '/templates'), {
method: 'POST',
headers: { Authorization: `Bearer ${accessToken}` },
body,
})

let data: any

try {
data = await res.json()
} catch (e) {
throw new Error(
`Build API request failed: ${res.statusText}`,
)
let res
if (templateID) {
res = await requestTemplateRebuild(accessToken, { templateID, ...args })
} else {
res = await requestTemplateBuild(accessToken, args)
}

if (!res.ok) {
const error:
| e2b.paths['/templates']['post']['responses']['401']['content']['application/json']
| e2b.paths['/templates']['post']['responses']['500']['content']['application/json'] =
data as any
res.data as any

if (error.code === 401) {
throw new Error(
Expand All @@ -483,5 +480,40 @@ async function buildTemplate(
)
}

return data as any
return res.data as any
}


async function triggerBuild(
accessToken: string,
templateID: string,
buildID: string,
) {
const res = await triggerTemplateBuild(accessToken, { templateID, buildID })

if (!res.ok) {
const error:
| e2b.paths['/templates/{templateID}/builds/{buildID}']['post']['responses']['401']['content']['application/json']
| e2b.paths['/templates/{templateID}/builds/{buildID}']['post']['responses']['500']['content']['application/json'] =
res.data as any

if (error.code === 401) {
throw new Error(
`Authentication error: ${res.statusText}, ${error.message ?? 'no message'
}`,
)
}

if (error.code === 500) {
throw new Error(
`Server error: ${res.statusText}, ${error.message ?? 'no message'}`,
)
}

throw new Error(
`API request failed: ${res.statusText}, ${error.message ?? 'no message'}`,
)
}

return
}
2 changes: 2 additions & 0 deletions packages/cli/src/commands/template/list.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export const listCommand = new commander.Command('list')
columns: [
{ name: 'templateID', alignment: 'left', title: 'Template ID' },
{ name: 'aliases', alignment: 'left', title: 'Template Name', color: 'orange' },
{ name: 'cpuCount', alignment: 'right', title: 'vCPUs' },
{ name: 'memoryMB', alignment: 'right', title: 'RAM MiB' },
],
disabledColumns: ['public', 'buildID'],
rows: templates.map((template) => ({ ...template, aliases: listAliases(template.aliases) })),
Expand Down
Loading

0 comments on commit 2bafaf0

Please sign in to comment.