Skip to content

Commit f99f6c1

Browse files
jmchiltonjmchilton
committed
Implement conda support.
This PR adds conda support to Planemo and via shared libraries to Galaxy in the form of new planemo commands providing a Galaxy tool-centric interface to conda for development and a Galaxy dependency resolver. Once the corresponding Galaxy PR is merged, the following command:: planemo t --conda_dependency_resolution --conda_auto_install --conda_auto_init bwa_and_samtools.xml Will test the supplied tool with only conda dependency resolution available. On startup, if Galaxy cannot find conda it will install it and for every dependency encountered it will attempt to find and install it on-demand. Planemo Conda Commands: - conda_init: Install a conda runtime. - conda_install: Takes in a tool or set of tools and installs their requirements using conda in such a way that they can be recovered and reused by dependency resolution. - conda_env: Build isolated environment for a tool or set of tools. Conda CLI Options: Common options for conda commands in planemo. --conda_prefix: Location of conda runtime (defaults to ~/miniconda2). --conda_exec: Location of conda executble (defaults to <conda_prefix/bin/conda) --conda_debug: Flag, when enabled conda will execute with the --debug flag. --conda_ensure_channels: Ensure channels are available (defaults to "r,bioconda") This PR implement a conda dependency resolver for Galaxy. This has a variety of options including: - copy_dependencies: This will copy dependencies into the working directory instead linking them - by passing --copy to conda create. Defaults to false. - auto_init: This will install conda if not avaiable on the system. Defaults to false. - auto_install: This will attempt to install packages for if they are not already installed. - Options mirroring the above for planemo - prefix, exec, debug, ensur_channels. All these dependency resolution option can be set as attributes on the dependency resolver element in dependency_resolvers_conf.xml.sample, or in galaxy.ini with the prefix conda_. Trying it out: $ planemo conda_init # install conda $ # setup a couple tools for conda testing $ planemo project_init --templates conda_test $ cd conda_test $ # look at tool requirements and install packages $ planemo conda_install bwa.xml $ # Load up the tool's environment to explore interactively $ . <(planemo conda_env bwa.xml) (bwa)$ which bwa /home/john/miniconda2/envs/jobdepsdpzBA...186c6a5504767f9e7/bin/bwa $ conda_env_deactivate $ # Run the tool in galaxy using conda dependency resolution $ # All the tool and test case do is verify the dependency and version $ planemo t --conda_dependency_resolution bwa.xml $ # The galaxy dependency resolver can even install dependencies on demand $ planemo t --conda_dependency_resolution --conda_auto_install bwa_and_samtools.xml
1 parent 4bccd2c commit f99f6c1

File tree

15 files changed

+900
-10
lines changed

15 files changed

+900
-10
lines changed

‎planemo/commands/cmd_brew_env.py‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from planemo.cli import pass_context
66
from planemo import options
7+
from planemo.io import ps1_for_path
78

89
from galaxy.tools.loader import load_tool
910
from galaxy.tools.deps.requirements import parse_requirements_from_xml
@@ -15,11 +16,7 @@
1516
@click.command('brew_env')
1617
@options.optional_tools_arg()
1718
@options.brew_option()
18-
@click.option(
19-
"--skip_install",
20-
is_flag=True,
21-
help="Skip installation - only source requirements already available."
22-
)
19+
@options.skip_install_option()
2320
@click.option(
2421
"--shell",
2522
is_flag=True
@@ -67,10 +64,9 @@ def cli(ctx, path, brew=None, skip_install=False, shell=None):
6764
# TODO: Would be cool if this wasn't a bunch of random hackery.
6865
launch_shell = os.environ.get("SHELL")
6966
if "bash" in launch_shell:
70-
file_name = os.path.basename(path)
71-
base_name = os.path.splitext(file_name)[0]
67+
ps1 = ps1_for_path(path)
7268
launch_shell = '(source ~/.bashrc; env PS1="%s" %s --norc)' % (
73-
"(%s)${PS1}" % base_name,
69+
ps1,
7470
launch_shell,
7571
)
7672
lines.extend([launch_shell])

‎planemo/commands/cmd_conda_env.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
from __future__ import print_function
2+
import click
3+
4+
from planemo.cli import pass_context
5+
from planemo import options
6+
7+
from planemo.io import ps1_for_path
8+
from planemo.io import error
9+
10+
from planemo.conda import build_conda_context, collect_conda_targets
11+
12+
from galaxy.tools.deps import conda_util
13+
14+
15+
SOURCE_COMMAND = """
16+
PRE_CONDA_PS1=$PS1
17+
source %s %s
18+
if [[ -n $BASH_VERSION ]]; then
19+
hash -r
20+
elif [[ -n $ZSH_VERSION ]]; then
21+
rehash
22+
else
23+
echo 'Only bash and zsh are supported'
24+
return 1
25+
fi
26+
PS1="%s"
27+
echo 'Deactivate environment with conda_env_deactivate'
28+
alias conda_env_deactivate="source %s; %s"
29+
"""
30+
31+
32+
@click.command('conda_env')
33+
@options.optional_tools_arg()
34+
@options.conda_target_options()
35+
# @options.skip_install_option() # TODO
36+
@pass_context
37+
def cli(ctx, path, **kwds):
38+
"""Source output to activate a conda environment for this tool.
39+
40+
% . <(planemo conda_env bowtie2.xml)
41+
% which bowtie2
42+
TODO_PLACE_PATH_HERE
43+
"""
44+
conda_context = build_conda_context(use_planemo_shell_exec=False, **kwds)
45+
conda_targets = collect_conda_targets(
46+
path, conda_context=conda_context
47+
)
48+
installed_conda_targets = conda_util.filter_installed_targets(
49+
conda_targets, conda_context=conda_context
50+
)
51+
env_name, exit_code = conda_util.build_isolated_environment(
52+
installed_conda_targets, conda_context=conda_context
53+
)
54+
if exit_code:
55+
error("Failed to build environmnt for request.")
56+
return 1
57+
58+
ps1 = ps1_for_path(path, base="PRE_CONDA_PS1")
59+
remove_env = "%s env remove -y --name '%s'" % (
60+
conda_context.conda_exec, env_name
61+
)
62+
deactivate = conda_context.deactivate
63+
activate = conda_context.activate
64+
command = SOURCE_COMMAND % (
65+
activate, env_name, ps1,
66+
deactivate, remove_env
67+
)
68+
print(command)

‎planemo/commands/cmd_conda_init.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
import click
2+
3+
from planemo.cli import pass_context
4+
from planemo.io import shell
5+
from planemo import options
6+
from planemo.conda import build_conda_context
7+
8+
from galaxy.tools.deps import conda_util
9+
10+
11+
@click.command('conda_init')
12+
@options.conda_target_options()
13+
@pass_context
14+
def cli(ctx, **kwds):
15+
"""Download and install conda.
16+
17+
This will download conda for managing dependencies for your platform
18+
using the appropriate Miniconda installer.
19+
20+
By running this command, you are agreeing to the terms of the conda
21+
license a 3-clause BSD 3 license. Please review full license at
22+
http://docs.continuum.io/anaconda/eula.
23+
"""
24+
conda_context = build_conda_context(**kwds)
25+
return conda_util.install_conda(conda_context=conda_context,
26+
shell_exec=shell)

‎planemo/commands/cmd_conda_install.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import click
2+
3+
from planemo.cli import pass_context
4+
from planemo.io import coalesce_return_codes
5+
from planemo import options
6+
7+
from planemo.conda import build_conda_context, collect_conda_targets
8+
9+
from galaxy.tools.deps import conda_util
10+
11+
12+
@click.command('conda_install')
13+
@options.optional_tools_arg()
14+
@options.conda_target_options()
15+
@pass_context
16+
def cli(ctx, path, **kwds):
17+
"""Install conda packages for tool requirements.
18+
"""
19+
conda_context = build_conda_context(**kwds)
20+
return_codes = []
21+
for conda_target in collect_conda_targets(path):
22+
ctx.log("Install conda target %s" % conda_target)
23+
return_code = conda_util.install_conda_target(
24+
conda_target, conda_context=conda_context
25+
)
26+
return_codes.append(return_code)
27+
return coalesce_return_codes(return_codes, assert_at_least_one=True)

‎planemo/conda.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
""" Planemo specific utilities for dealing with conda, extending Galaxy's
2+
features with planemo specific idioms.
3+
"""
4+
5+
from galaxy.tools.deps import conda_util
6+
from planemo.io import shell
7+
8+
from galaxy.tools.deps.requirements import parse_requirements_from_xml
9+
from galaxy.tools.loader_directory import load_tool_elements_from_path
10+
11+
12+
def build_conda_context(**kwds):
13+
""" Build a Galaxy CondaContext tailored to planemo use
14+
and common command-line arguments.
15+
"""
16+
conda_prefix = kwds.get("conda_prefix", None)
17+
use_planemo_shell = kwds.get("use_planemo_shell_exec", True)
18+
ensure_channels = kwds.get("conda_ensure_channels", "")
19+
shell_exec = shell if use_planemo_shell else None
20+
return conda_util.CondaContext(conda_prefix=conda_prefix,
21+
ensure_channels=ensure_channels,
22+
shell_exec=shell_exec)
23+
24+
25+
def collect_conda_targets(path, found_tool_callback=None, conda_context=None):
26+
conda_targets = []
27+
for (tool_path, tool_xml) in load_tool_elements_from_path(path):
28+
if found_tool_callback:
29+
found_tool_callback(tool_path)
30+
requirements, containers = parse_requirements_from_xml(tool_xml)
31+
conda_targets.extend(conda_util.requirements_to_conda_targets(requirements))
32+
return conda_targets

‎planemo/galaxy_config.py‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,13 @@
6161
</job_metrics>
6262
"""
6363

64+
# TODO: fill in properties to match CLI args.
65+
CONDA_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
66+
<conda ${attributes} />
67+
<conda versionless="true" ${attributes} />
68+
</dependency_resolvers>
69+
"""
70+
6471

6572
BREW_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
6673
<homebrew />
@@ -84,6 +91,7 @@
8491
STOCK_DEPENDENCY_RESOLUTION_STRATEGIES = {
8592
"brew_dependency_resolution": BREW_DEPENDENCY_RESOLUTION_CONF,
8693
"shed_dependency_resolution": SHED_DEPENDENCY_RESOLUTION_CONF,
94+
"conda_dependency_resolution": CONDA_DEPENDENCY_RESOLUTION_CONF,
8795
}
8896

8997
EMPTY_TOOL_CONF_TEMPLATE = """<toolbox></toolbox>"""
@@ -649,6 +657,7 @@ def _handle_dependency_resolution(config_directory, kwds):
649657
"brew_dependency_resolution",
650658
"dependency_resolvers_config_file",
651659
"shed_dependency_resolution",
660+
"conda_dependency_resolution",
652661
]
653662

654663
selected_strategies = 0
@@ -660,13 +669,36 @@ def _handle_dependency_resolution(config_directory, kwds):
660669
message = "At most one option from [%s] may be specified"
661670
raise click.UsageError(message % resolutions_strategies)
662671

672+
dependency_attribute_kwds = {
673+
'conda_prefix': None,
674+
'conda_exec': None,
675+
'conda_debug': False,
676+
'conda_copy_dependencies': False,
677+
'conda_auto_init': False,
678+
'conda_auto_install': False,
679+
'conda_ensure_channels': '',
680+
}
681+
682+
attributes = []
683+
for key, default_value in dependency_attribute_kwds.iteritems():
684+
value = kwds.get(key, default_value)
685+
if value != default_value:
686+
# Strip leading prefix (conda_) off attributes
687+
attribute_key = "_".join(key.split("_")[1:])
688+
attributes.append('%s="%s"' % (attribute_key, value))
689+
690+
attribute_str = " ".join(attributes)
691+
663692
for key in STOCK_DEPENDENCY_RESOLUTION_STRATEGIES:
664693
if kwds.get(key):
665694
resolvers_conf = os.path.join(
666695
config_directory,
667696
"resolvers_conf.xml"
668697
)
669-
conf_contents = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
698+
template_str = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
699+
conf_contents = Template(template_str).safe_substitute({
700+
'attributes': attribute_str
701+
})
670702
open(resolvers_conf, "w").write(conf_contents)
671703
kwds["dependency_resolvers_config_file"] = resolvers_conf
672704

‎planemo/io.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,16 @@ def temp_directory(prefix="planemo_tmp_"):
101101
shutil.rmtree(temp_dir)
102102

103103

104+
def ps1_for_path(path, base="PS1"):
105+
""" Used by environment commands to build a PS1 shell
106+
variables for tool or directory of tools.
107+
"""
108+
file_name = os.path.basename(path)
109+
base_name = os.path.splitext(file_name)[0]
110+
ps1 = "(%s)${%s}" % (base_name, base)
111+
return ps1
112+
113+
104114
def kill_pid_file(pid_file):
105115
if not os.path.exists(pid_file):
106116
return

0 commit comments

Comments
 (0)