First crack at containers documentation and related usability improvements.

jmchilton · jmchilton · commit 0a1abfea5352 · 2017-03-03T16:38:41.000-05:00
- Implement ``planemo lint --biocontainer`` to mirror ``planemo lint --conda_requirements``. Just checks that a tool is ready to go with best practice namespaces for BioContainers tools.
- Implement test case for new linting flag.
- Add new section of documentation to advanced tool topics for containers. Work through examples of both an existing working container (seqtk) and one that must be built on the fly. Demonstrate ``planemo mull``.
- Hack around a bug in Conda 4.2 that makes it so ``planemo mull`` doesn't work out of the box on Mac OS X.
- Add more options and more documentation to the ``planemo mull`` command.
diff --git a/docs/_writing_dependencies_conda.rst b/docs/_writing_dependencies_conda.rst
@@ -1,3 +1,5 @@
+.. _dependencies_and_conda:
+
 Dependencies and Conda
 ===========================================
 
diff --git a/docs/_writing_dependencies_docker.rst b/docs/_writing_dependencies_docker.rst
diff --git a/docs/conf.py b/docs/conf.py
@@ -86,7 +86,7 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build']
+exclude_patterns = ['_build', '_*']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
diff --git a/docs/planemo.commands.rst b/docs/planemo.commands.rst
@@ -68,6 +68,14 @@ planemo.commands.cmd_clone module
     :undoc-members:
     :show-inheritance:
 
+planemo.commands.cmd_conda_build module
+---------------------------------------
+
+.. automodule:: planemo.commands.cmd_conda_build
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 planemo.commands.cmd_conda_env module
 -------------------------------------
 
diff --git a/docs/planemo.linters.rst b/docs/planemo.linters.rst
@@ -4,6 +4,14 @@ planemo.linters package
 Submodules
 ----------
 
+planemo.linters.biocontainer_registered module
+----------------------------------------------
+
+.. automodule:: planemo.linters.biocontainer_registered
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 planemo.linters.conda_requirements module
 -----------------------------------------
 
diff --git a/docs/writing_advanced.rst b/docs/writing_advanced.rst
@@ -14,6 +14,7 @@ available - check out tutorial_ if you have never developed a Galaxy tool.
 .. include:: _writing_tool_metadata.rst
 .. include:: _writing_clusters.rst
 .. include:: _writing_dependencies_conda.rst
+.. include:: _writing_dependencies_docker.rst
 
 .. _Galaxy: http://galaxyproject.org/
 .. _Docker: https://www.docker.com/
diff --git a/planemo/commands/cmd_docker_shell.py b/planemo/commands/cmd_docker_shell.py
@@ -64,6 +64,7 @@ def cli(ctx, path, **kwds):
         if container.type == "docker":
             identifier = container.identifier
 
+    # Refactor mulled container resolver to be able to do this.
     if kwds["from_tag"]:
         identifier = "-t %s" % identifier
 
diff --git a/planemo/commands/cmd_lint.py b/planemo/commands/cmd_lint.py
@@ -33,6 +33,12 @@
     default=False,
     help="Check tool requirements for availability in best practice Conda channels.",
 )
+@click.option(
+    "--biocontainer",
+    is_flag=True,
+    default=False,
+    help="Check best practice BioContainer namespaces for a container definition applicable for this tool.",
+)
 # @click.option(
 # "--verify",
 # is_flag=True,
diff --git a/planemo/commands/cmd_mull.py b/planemo/commands/cmd_mull.py
@@ -13,6 +13,8 @@
 @click.command('mull')
 @options.optional_tools_arg(multiple=True)
 @options.recursive_option()
+@options.mulled_options()
+@options.conda_ensure_channels_option()
 @command_function
 def cli(ctx, paths, **kwds):
     """Build containers for specified tools.
@@ -30,4 +32,5 @@ def cli(ctx, paths, **kwds):
     for conda_targets in collect_conda_target_lists(ctx, paths):
         mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
         mull_target_kwds = build_mull_target_kwds(ctx, **kwds)
-        mull_targets(mulled_targets, command="build", **mull_target_kwds)
+        command = kwds["mulled_command"]
+        mull_targets(mulled_targets, command=command, **mull_target_kwds)
diff --git a/planemo/lint.py b/planemo/lint.py
@@ -15,6 +15,7 @@
     urlopen,
 )
 
+import planemo.linters.biocontainer_registered
 import planemo.linters.conda_requirements
 import planemo.linters.doi
 import planemo.linters.urls
@@ -60,6 +61,9 @@ def _lint_extra_modules(**kwds):
     if kwds.get("conda_requirements", False):
         linters.append(planemo.linters.conda_requirements)
 
+    if kwds.get("biocontainer", False):
+        linters.append(planemo.linters.biocontainer_registered)
+
     return linters
 
 
diff --git a/planemo/linters/biocontainer_registered.py b/planemo/linters/biocontainer_registered.py
@@ -0,0 +1,60 @@
+"""Ensure best-practice biocontainer registered for this tool."""
+
+from galaxy.tools.deps.mulled.util import (
+    build_target,
+    image_name,
+    mulled_tags_for,
+    split_tag,
+)
+
+from planemo.conda import tool_source_conda_targets
+
+MESSAGE_WARN_NO_REQUIREMENTS = "No valid package requirement tags found to infer BioContainer from."
+MESSAGE_WARN_NO_CONTAINER = "Failed to find a BioContainer registered for these requirements."
+MESSAGE_INFO_FOUND_BIOCONTAINER = "BioContainer best-practice container found [%s]."
+
+
+def lint_biocontainer_registered(tool_source, lint_ctx):
+    conda_targets = tool_source_conda_targets(tool_source)
+    if not conda_targets:
+        lint_ctx.warn(MESSAGE_WARN_NO_REQUIREMENTS)
+        return
+
+    mulled_targets = map(lambda c: build_target(c.package, c.version), conda_targets)
+    name = mulled_container_name("biocontainers", mulled_targets)
+    if name:
+        lint_ctx.info(MESSAGE_INFO_FOUND_BIOCONTAINER % name)
+    else:
+        lint_ctx.warn(MESSAGE_WARN_NO_CONTAINER)
+
+
+# TODO: Refactor following method into mulled util and then refactor same code out of mulled
+# container resolver.
+def mulled_container_name(namespace, targets):
+    name = None
+
+    if len(targets) == 1:
+        target = targets[0]
+        target_version = target.version
+        tags = mulled_tags_for(namespace, target.package_name)
+
+        if not tags:
+            return None
+
+        if target_version:
+            for tag in tags:
+                version, build = split_tag(tag)
+                if version == target_version:
+                    name = "%s:%s--%s" % (target.package_name, version, build)
+                    break
+        else:
+            version, build = split_tag(tags[0])
+            name = "%s:%s--%s" % (target.package_name, version, build)
+    else:
+        base_image_name = image_name(targets)
+        tags = mulled_tags_for(namespace, base_image_name)
+        if tags:
+            name = "%s:%s" % (base_image_name, tags[0])
+
+    if name:
+        return "quay.io/%s/%s" % (namespace, name)
diff --git a/planemo/mulled.py b/planemo/mulled.py
@@ -12,7 +12,7 @@
     InvolucroContext,
 )
 
-from planemo.io import shell
+from planemo.io import IS_OS_X, shell
 
 
 def build_involucro_context(ctx, **kwds):
@@ -35,13 +35,26 @@ def build_mull_target_kwds(ctx, **kwds):
     """Adapt Planemo's CLI and workspace configuration to galaxy-lib's mulled_build options."""
     involucro_context = build_involucro_context(ctx, **kwds)
     channels = kwds.get("conda_ensure_channels", ",".join(DEFAULT_CHANNELS))
-
-    return {
+    namespace = kwds.get("mulled_namespace", "biocontainers")
+    target_kwds = {
         'involucro_context': involucro_context,
         'channels': channels.split(","),
+        'namespace': namespace,
+        'verbose': ctx.verbose,
     }
 
+    conda_version = kwds.get("mulled_conda_version", None)
+    if conda_version is not None:
+        target_kwds["conda_version"] = conda_version
+    else:
+        # Hack to workaround a bug with osxfs + Conda 4.2 - remove
+        # when container gets upgraded to 4.3
+        if IS_OS_X:
+            target_kwds["conda_version"] = "4.3"
+    return target_kwds
+
 
 __all__ = (
     "build_involucro_context",
+    "build_mull_target_kwds",
 )
diff --git a/planemo/options.py b/planemo/options.py
@@ -1538,3 +1538,44 @@ def tool_init_example_command_option(help=EXAMPLE_COMMAND_HELP):
         prompt=False,
         help=help,
     )
+
+
+def mulled_conda_option():
+    return planemo_option(
+        "--mulled_conda_version",
+        type=click.STRING,
+        default=None,
+        help=("Install a specific version of Conda before running the command, by "
+              "default the version that comes with the continuumio miniconda3 image "
+              "will be used under Linux and under Mac OS X Conda will be upgraded to "
+              "to work around a bug in 4.2.")
+    )
+
+
+def mulled_namespace_option():
+    return planemo_option(
+        "--mulled_namespace",
+        type=click.STRING,
+        default="biocontainers",
+        help=("Build a mulled image with the specified namespace - defaults to "
+              "biocontainers. Galaxy currently only recognizes images with the "
+              "namespace biocontainers.")
+    )
+
+
+def mulled_action_option():
+    return planemo_option(
+        "--mulled_command",
+        type=click.STRING,
+        default="build",
+        help=("Mulled action to perform for targets - this defaults to 'build'. "
+              "Set this to build-and-test to also test the resulting container.")
+    )
+
+
+def mulled_options():
+    return _compose(
+        mulled_conda_option(),
+        mulled_namespace_option(),
+        mulled_action_option(),
+    )
diff --git a/project_templates/conda_testing/bwa_and_samtools.xml b/project_templates/conda_testing/bwa_and_samtools.xml
@@ -1,7 +1,7 @@
 <tool id="bwa_and_samtools" name="BWA and samtools" version="0.1.0">
     <requirements>
-        <requirement type="package" version="0.7.12">bwa</requirement>
-        <requirement type="package" version="1.2">samtools</requirement>
+        <requirement type="package" version="0.7.15">bwa</requirement>
+        <requirement type="package" version="1.3.1">samtools</requirement>
     </requirements>
     <stdio>
         <exit_code range="2:" />
@@ -23,12 +23,12 @@
       <test>
         <output name="output_1">
           <assert_contents>
-            <has_text text="Version: 0.7.12-r1039" />
+            <has_text text="Version: 0.7.15-r1140" />
           </assert_contents>
         </output>
         <output name="output_2">
           <assert_contents>
-            <has_text text="Version: 1.2" />
+            <has_text text="Version: 1.3.1" />
           </assert_contents>
         </output>
       </test>
diff --git a/requirements.txt b/requirements.txt
@@ -13,6 +13,6 @@ virtualenv
 lxml
 gxformat2>=0.1.1
 ephemeris>=0.2.0
-galaxy-lib>=17.5.5
+galaxy-lib>=17.5.6
 html5lib>=0.9999999,!=0.99999999,!=0.999999999,!=1.0b10,!=1.0b09 ; python_version == '2.7'
 cwltool==1.0.20160726135535 ; python_version == '2.7'
diff --git a/tests/test_lint.py b/tests/test_lint.py
@@ -91,6 +91,5 @@ def test_lint_dependencies_okay(self):
         self._check_exit_code(lint_cmd)
         lint_cmd = ["lint", "--conda_requirements", seqtk_seq_v6]
         self._check_exit_code(lint_cmd)
-        # Doesn't work, it seems to be missing?
-        # lint_cmd = ["lint", "--biocontainer_registered", bwa_no_reqs]
-        # self._check_exit_code(lint_cmd)
+        lint_cmd = ["lint", "--biocontainer", seqtk_seq_v6]
+        self._check_exit_code(lint_cmd)

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+.. _dependencies_and_conda:`
	`2`	`+`
`1`	`3`	`Dependencies and Conda`
`2`	`4`	`===========================================`
`3`	`5`