Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

Commit 392203b

Browse files
beneboybeneboy
committed
feat: Add /execute endpoint and docker refactorings
1 parent b84d204 commit 392203b

File tree

6 files changed

+170
-34
lines changed

6 files changed

+170
-34
lines changed

‎envs/nodejs.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
'@context': 'https://stenci.la/schema/v1/'
22
type: Environment
33
name: nodejs
4-
description: |
5-
Contains only Node.js, the open-source, cross-platform JavaScript run-time environment.
4+
description: >
5+
Contains only Node.js, the open-source, cross-platform JavaScript run-time
6+
environment.
67
adds:
78
- nodejs

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
"dependencies": {
6565
"await-spawn": "^2.1.2",
6666
"better-sqlite3": "^5.2.1",
67+
"body-parser": "^1.18.3",
6768
"chalk": "^2.4.2",
6869
"del": "^3.0.0",
6970
"ellipsize": "^0.1.0",

‎src/Environment.ts‎

Lines changed: 111 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ export enum Platform {
2121
UNIX, WIN, DOCKER
2222
}
2323

24+
const DOCKER_DEFAULT_COMMAND = 'sh'
25+
const DOCKER_CONTAINER_ID_SHORT_LENGTH = 12
26+
2427
/**
2528
* Parameters of a user session inside an environment
2629
*/
@@ -49,6 +52,25 @@ export class SessionParameters {
4952
* Standard output stream
5053
*/
5154
stdout: stream.Writable = process.stdout
55+
56+
/**
57+
* CPU shares (only applies when using Docker platform). Priority of the container relative to other processes.
58+
* The default is 1024, a higher number means higher priority for execution (when CPU contention exists).
59+
*/
60+
cpuShares: Number = 1024
61+
62+
/**
63+
* Memory limit (only applies when using Docker platform). Maximum amount of memory a container can use.
64+
* Should be set in the format <number>[<unit>]). Number is a positive integer. Unit can be one of b, k, m, or g.
65+
* Minimum is 4M.
66+
*/
67+
memoryLimit: string = '0'
68+
69+
/**
70+
* The ID of the container to attempt to use for an execution. It may be an empty string in which case the executor
71+
* will start a new container.
72+
*/
73+
containerId: string = ''
5274
}
5375

5476
/**
@@ -350,11 +372,11 @@ export default class Environment {
350372
/**
351373
* Create variables for an environment.
352374
*
353-
* This method is used in several other metho
375+
* This method is used in several other methods
354376
* e.g. `within`, `enter`
355377
*
356378
* A 'pure' environment will only have available the executables that
357-
* were exlicitly installed into the environment
379+
* were explicitly installed into the environment
358380
*
359381
* @param pure Should the shell that this command is executed in be 'pure'?
360382
*/
@@ -389,14 +411,42 @@ export default class Environment {
389411
})
390412
}
391413

414+
private async getDockerShellArgs (dockerCommand: string, sessionParameters: SessionParameters, daemonize: boolean = false): Promise<Array<string>> {
415+
const { command, cpuShares, memoryLimit } = sessionParameters
416+
const nixLocation = await nix.location(this.name)
417+
const shellArgs = [
418+
dockerCommand, '--interactive', '--tty', '--rm',
419+
// Prepend the environment path to the PATH variable
420+
'--env', `PATH=${nixLocation}/bin:${nixLocation}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
421+
// We also need to tell R where to find libraries
422+
'--env', `R_LIBS_SITE=${nixLocation}/library`,
423+
// Read-only bind mount of the Nix store
424+
'--volume', '/nix/store:/nix/store:ro',
425+
// Apply CPU shares
426+
`--cpu-shares=${cpuShares}`,
427+
// Apply memory limit
428+
`--memory=${memoryLimit}`,
429+
// We use Alpine Linux as a base image because it is very small but has some basic
430+
// shell utilities (lkike ls and uname) that are good for debugging but also sometimes
431+
// required for things like R
432+
'alpine'
433+
].concat(
434+
// Command to execute in the container
435+
command ? command.split(' ') : DOCKER_DEFAULT_COMMAND
436+
)
437+
438+
if (daemonize) shellArgs.splice(1, 0, '-d')
439+
440+
return shellArgs
441+
}
442+
392443
/**
393444
* Enter the a shell within the environment
394445
*
395446
* @param sessionParameters Parameters of the session
396447
*/
397448
async enter (sessionParameters: SessionParameters) {
398449
let { command, platform, pure, stdin, stdout } = sessionParameters
399-
const location = await nix.location(this.name)
400450

401451
if (platform === undefined) {
402452
switch (os.platform()) {
@@ -417,22 +467,7 @@ export default class Environment {
417467
break
418468
case Platform.DOCKER:
419469
shellName = 'docker'
420-
shellArgs = [
421-
'run', '--interactive', '--tty', '--rm',
422-
// Prepend the environment path to the PATH variable
423-
'--env', `PATH=${location}/bin:${location}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`,
424-
// We also need to tell R where to find libraries
425-
'--env', `R_LIBS_SITE=${location}/library`,
426-
// Read-only bind mount of the Nix store
427-
'--volume', '/nix/store:/nix/store:ro',
428-
// We use Alpine Linux as a base image because it is very small but has some basic
429-
// shell utilities (lkike ls and uname) that are good for debugging but also sometimes
430-
// required for things like R
431-
'alpine'
432-
].concat(
433-
// Command to execute in the container
434-
command ? command.split(' ') : 'sh'
435-
)
470+
shellArgs = await this.getDockerShellArgs('run', sessionParameters, false)
436471
break
437472
default:
438473
shellName = 'bash'
@@ -452,10 +487,14 @@ export default class Environment {
452487
// During development you'll need to use ---pure=false so that
453488
// node is available to run Nixster. In production, when a user
454489
// has installed a binary, this shouldn't be necessary
455-
let nixsterPath = await spawn('which', ['nixster'])
456-
const tempRcFile = tmp.fileSync()
457-
fs.writeFileSync(tempRcFile.name, `alias nixster="${nixsterPath.toString().trim()}"\n`)
458-
shellArgs.push('--rcfile', tempRcFile.name)
490+
try {
491+
let nixsterPath = await spawn('which', ['nixster'])
492+
const tempRcFile = tmp.fileSync()
493+
fs.writeFileSync(tempRcFile.name, `alias nixster="${nixsterPath.toString().trim()}"\n`)
494+
shellArgs.push('--rcfile', tempRcFile.name)
495+
} catch (e) {
496+
// ignore
497+
}
459498
}
460499

461500
// Environment variables
@@ -512,4 +551,53 @@ export default class Environment {
512551

513552
if (platform === Platform.UNIX && command) shellProcess.write(command + '\r')
514553
}
554+
555+
private async checkContainerRunning (containerId: string) {
556+
const containerRegex = new RegExp(/^[^_\W]{12}$/)
557+
if (containerRegex.exec(containerId) === null) {
558+
throw new Error(`'${containerId}' is not a valid docker container ID.`)
559+
}
560+
561+
// List running containers that match the containerId we are looking for. There should be only one or zero.
562+
const dockerPsProcess = await spawn('docker', ['ps', '-q', '--filter', `id=${containerId}`])
563+
const foundContainerId = dockerPsProcess.toString().trim()
564+
return foundContainerId === containerId // foundContainerId should be either containerId or an empty string
565+
}
566+
567+
/**
568+
* Start a new Docker container and execute a command within it. The container daemonizes and keeps running
569+
* (until the process it is running stops).
570+
*
571+
* Returns the short ID of the container that is running.
572+
*/
573+
async execute (sessionParameters: SessionParameters): Promise<string> {
574+
if (sessionParameters.platform !== Platform.DOCKER) {
575+
throw new Error('Execute is only valid with the Docker platform.')
576+
}
577+
578+
const shellArgs = await this.getDockerShellArgs('run', sessionParameters, true)
579+
580+
const dockerProcess = await spawn('docker', shellArgs)
581+
return dockerProcess.toString().trim().substr(0, DOCKER_CONTAINER_ID_SHORT_LENGTH)
582+
}
583+
584+
/**
585+
* Build a Docker container for this environment
586+
*/
587+
async dockerBuild () {
588+
const requisites = await nix.requisites(this.name)
589+
const dockerignore = `*\n${requisites.map(req => '!' + req).join('\n')}`
590+
console.log(dockerignore)
591+
592+
// The Dockerfile does essentially the same as the `docker run` command
593+
// generated above in `dockerRun`...
594+
const location = await nix.location(this.name)
595+
const dockerfile = `
596+
FROM alpine
597+
ENV PATH ${location}/bin:${location}/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
598+
ENV R_LIBS_SITE=${location}/library
599+
COPY /nix/store /nix/store
600+
`
601+
console.log(dockerfile)
602+
}
515603
}

‎src/cli.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,12 @@ yargs
177177
alias: 'env',
178178
type: 'string'
179179
})
180+
.option('docker', {
181+
describe: 'Build the environment using Docker? Nix packages are stored on the host.',
182+
type: 'boolean',
183+
alias: 'd',
184+
default: false
185+
})
180186
.env('NIXSTER')
181187
.option('docker', {
182188
describe: 'Also build a Docker container for the environment?',

‎src/nix.ts‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ export function semver (version: string): string {
2121
let match = version.match(/^(\d+)(\.(\d+))?(\.(\d+))?(.*)?/)
2222
return match ? sprintf('%05i.%05i.%05i%s', match[1], match[3] || 0, match[5] || 0, match[6] || '') : version
2323
}
24+
2425
// Register function in the database
2526
try {
2627
db.function('semver', semver)
@@ -218,7 +219,7 @@ export async function match (pkg: string): Promise<Array<any>> {
218219
* @param limit Limit on number of packages to return
219220
*/
220221
export async function search (term: string, type: string = '', limit: number = 1000): Promise<Array<any>> {
221-
term = term.replace("'", "\'")
222+
term = term.replace('\'', '\'')
222223
// TODO: find a way to show the channel and description for the latest
223224
// version
224225
const stmt = db.prepare(`
@@ -246,19 +247,18 @@ export async function search (term: string, type: string = '', limit: number = 1
246247
* @param env The environment name
247248
*/
248249
export async function built (env: string): Promise<boolean> {
249-
return (await location(env)).length > 0
250+
return (location(env)).length > 0
250251
}
251252

252253
/**
253254
* Get the location of an environment within the Nix store
254255
*
255256
* @param env The environment name
256257
*/
257-
export async function location (env: string): Promise<string> {
258+
export function location (env: string): string {
258259
const profile = path.join(profiles, env)
259260
if (!fs.existsSync(profile)) return ''
260-
const readlink = await spawn('readlink', ['-f', profile])
261-
return readlink
261+
return fs.realpathSync(profile)
262262
}
263263

264264
/**
@@ -267,8 +267,8 @@ export async function location (env: string): Promise<string> {
267267
* @param env The environment name
268268
* @param pkgs An array of normalized package names
269269
*/
270-
export async function install (env: string, pkgs: Array<string>, clean: boolean = false) {
271-
let channels: {[key: string]: any} = {}
270+
export async function install (env: string, pkgs: Array<string>, clean: boolean = false, inDocker: boolean = false) {
271+
let channels: { [key: string]: any } = {}
272272
for (let pkg of pkgs) {
273273
let matches = await match(pkg)
274274
if (matches.length === 0) {
@@ -295,7 +295,23 @@ export async function install (env: string, pkgs: Array<string>, clean: boolean
295295
if (clean) args = args.concat('--remove-all')
296296
args = args.concat('--attr', channels[channel].map((pkg: any) => pkg.attr))
297297

298-
await spawn('nix-env', args, {
298+
let command
299+
300+
if (inDocker) {
301+
command = 'docker'
302+
let dockerArgs = [
303+
'run',
304+
'stencila/nixster', 'nix-env'
305+
// todo: write correct commands
306+
]
307+
args = dockerArgs.concat(args)
308+
} else {
309+
command = 'nix-env'
310+
}
311+
312+
console.log(args.join(' '))
313+
314+
await spawn(command, args, {
299315
stdio: 'inherit'
300316
})
301317
}
@@ -363,7 +379,7 @@ export async function requisites (env: string): Promise<Array<any>> {
363379
const query = await spawn('nix-store', [
364380
'--query',
365381
'--requisites',
366-
await location(env)
382+
location(env)
367383
])
368384
const list = query.toString().trim()
369385
return list.length ? list.split('\n') : []

‎src/serve.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ const expressWs = require('express-ws')(app)
1111
// Serve static assets from ./static
1212
app.use(express.static(path.join(__dirname, 'static')))
1313

14+
// JSON Body Parsing
15+
const jsonParser = require('body-parser').json()
16+
app.use(jsonParser)
17+
18+
// todo: rename shell to interact
1419
// Instantiate shell and set up data handlers
1520
expressWs.app.ws('/shell', async (ws: any, req: express.Request) => {
1621
try {
@@ -51,5 +56,24 @@ app.use((error: Error, req: express.Request, res: express.Response, next: any) =
5156
next(error)
5257
})
5358

59+
expressWs.app.post('/execute', jsonParser, async (req: any, res: any) => {
60+
// req: some JSON -> new SessionParameters object to start in env.execute (new execute method)
61+
if (!req.body) return res.sendStatus(400)
62+
63+
const env = new Environment(req.body.environmentId)
64+
const sessionParameters = new SessionParameters()
65+
sessionParameters.platform = Platform.DOCKER
66+
sessionParameters.command = req.body.command || ''
67+
68+
const containerId = await env.execute(sessionParameters)
69+
res.status(200).json({
70+
containerId: containerId
71+
})
72+
})
73+
74+
expressWs.app.post('/stop', async (req: any, res: any) => {
75+
// req: some JSON -> with container ID that will stop the container
76+
})
77+
5478
app.listen(3000)
5579
console.error('Listening on http://localhost:3000')

0 commit comments

Comments
 (0)