Enable caching on --install_galaxy by default.

jmchilton · jmchilton · commit d755fe7fe4d0 · 2015-03-24T16:42:54.000-04:00
It now clones Galaxy on the first use to ~/.planemo/gx_repo and fetches updates on subsequent uses. Added new option --no_cache_galaxy to use the previous speedier download option (more appropriate for one time use).

Eggs are still downloaded each time - and probably take longer(?).
diff --git a/planemo/cli.py b/planemo/cli.py
@@ -25,14 +25,17 @@
 class Context(object):
 
     def __init__(self):
-        self.verbose = False
         self.home = os.getcwd()
         self._global_config = None
+        # Will be set by planemo CLI driver
+        self.verbose = False
+        self.planemo_config = None
+        self.planemo_directory = None
 
     @property
     def global_config(self):
         if self._global_config is None:
-            self._global_config = read_global_config()
+            self._global_config = read_global_config(self.planemo_config)
         return self._global_config
 
     def log(self, msg, *args):
@@ -46,6 +49,19 @@ def vlog(self, msg, *args):
         if self.verbose:
             self.log(msg, *args)
 
+    @property
+    def workspace(self):
+        if not self.planemo_directory:
+            raise Exception("No planemo workspace defined.")
+        workspace = self.planemo_directory
+        if not os.path.exists(workspace):
+            os.makedirs(workspace)
+        if not os.path.isdir(workspace):
+            template = "Planemo workspace directory [%s] unavailable."
+            message = template % workspace
+            raise Exception(message)
+        return workspace
+
 
 pass_context = click.make_pass_decorator(Context, ensure=True)
 cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -89,7 +105,15 @@ def get_command(self, ctx, name):
 @click.version_option(__version__)
 @click.option('-v', '--verbose', is_flag=True,
               help='Enables verbose mode.')
+@click.option('--config',
+              default="~/.planemo.yml",
+              help="Planemo configuration YAML file.")
+@click.option('--directory',
+              default="~/.planemo",
+              help="Workspace for planemo.")
 @pass_context
-def planemo(ctx, verbose):
+def planemo(ctx, config, directory, verbose):
     """Utilities to assist with the development of Galaxy tools."""
     ctx.verbose = verbose
+    ctx.planemo_config = os.path.expanduser(config)
+    ctx.planemo_directory = os.path.expanduser(directory)
diff --git a/planemo/commands/cmd_serve.py b/planemo/commands/cmd_serve.py
@@ -9,6 +9,7 @@
 @options.optional_tools_arg()
 @options.galaxy_root_option()
 @options.install_galaxy_option()
+@options.no_cache_galaxy_option()
 @options.no_cleanup_option()
 @options.test_data_option()
 @options.dependency_resolvers_option()
diff --git a/planemo/commands/cmd_test.py b/planemo/commands/cmd_test.py
@@ -84,6 +84,7 @@
 )
 @options.galaxy_root_option()
 @options.install_galaxy_option()
+@options.no_cache_galaxy_option()
 @options.no_cleanup_option()
 @options.test_data_option()
 @options.tool_data_table_option()
diff --git a/planemo/commands/cmd_tool_factory.py b/planemo/commands/cmd_tool_factory.py
@@ -9,6 +9,7 @@
 @click.command('tool_factory')
 @options.galaxy_root_option()
 @options.install_galaxy_option()
+@options.no_cache_galaxy_option()
 @options.no_cleanup_option()
 @options.test_data_option()
 @options.dependency_resolvers_option()
diff --git a/planemo/config.py b/planemo/config.py
@@ -5,17 +5,18 @@
 }
 
 
-def global_config_path():
-    config_path = os.environ.get(
-        "PLANEMO_GLOBAL_CONFIG_PATH",
-        "~/.planemo.yml"
-    )
-    config_path = os.path.expanduser(config_path)
+def global_config_path(config_path):
+    if not config_path:
+        config_path = os.environ.get(
+            "PLANEMO_GLOBAL_CONFIG_PATH",
+            "~/.planemo.yml"
+        )
+        config_path = os.path.expanduser(config_path)
     return config_path
 
 
-def read_global_config():
-    config_path = global_config_path()
+def read_global_config(config_path):
+    config_path = global_config_path(config_path)
     if not os.path.exists(config_path):
         return DEFAULT_CONFIG
 
diff --git a/planemo/galaxy_config.py b/planemo/galaxy_config.py
@@ -103,7 +103,7 @@ def config_join(*args):
         config_directory = mkdtemp()
     try:
         latest_galaxy = False
-        if _install_galaxy_if_needed(config_directory, kwds):
+        if _install_galaxy_if_needed(ctx, config_directory, kwds):
             latest_galaxy = True
             galaxy_root = config_join("galaxy-dev")
 
@@ -309,26 +309,52 @@ def _tool_conf_entry_for(tool_path):
     return tool_definition
 
 
-def _install_galaxy_if_needed(config_directory, kwds):
+def _install_galaxy_if_needed(ctx, config_directory, kwds):
     installed = False
     if kwds.get("install_galaxy", None):
-        _install_galaxy_via_download(config_directory, kwds)
+        if not kwds.get("no_cache_galaxy", False):
+            _install_galaxy_via_git(ctx, config_directory, kwds)
+        else:
+            _install_galaxy_via_download(config_directory, kwds)
         installed = True
     return installed
 
 
 def _install_galaxy_via_download(config_directory, kwds):
+    command = galaxy_run.DOWNLOAD_GALAXY + "; tar -zxvf dev | tail"
+    _install_with_command(config_directory, command)
+
+
+def _install_galaxy_via_git(ctx, config_directory, kwds):
+    _ensure_galaxy_repository_available(ctx)
+    workspace = ctx.workspace
+    gx_repo = os.path.join(workspace, "gx_repo")
+    command = "git clone %s galaxy-dev" % (gx_repo)
+    _install_with_command(config_directory, command)
+
+
+def _install_with_command(config_directory, command):
     install_cmds = [
         "cd %s" % config_directory,
-        galaxy_run.DOWNLOAD_GALAXY,
-        "tar -zxvf master | tail",
+        command,
         "cd galaxy-dev",
         "type virtualenv >/dev/null 2>&1 && virtualenv .venv",
         galaxy_run.ACTIVATE_COMMAND,
     ]
     shell(";".join(install_cmds))
 
 
+def _ensure_galaxy_repository_available(ctx):
+    workspace = ctx.workspace
+    gx_repo = os.path.join(workspace, "gx_repo")
+    if os.path.exists(gx_repo):
+        # Attempt fetch - but don't fail if not interweb, etc...
+        shell("git --git-dir %s fetch >/dev/null 2>&1" % gx_repo)
+    else:
+        remote_repo = "https://github.com/galaxyproject/galaxy"
+        shell("git clone --bare %s %s" % (remote_repo, gx_repo))
+
+
 def _build_env_for_galaxy(properties, template_args):
     env = {}
     for key, value in properties.iteritems():
diff --git a/planemo/options.py b/planemo/options.py
@@ -92,6 +92,17 @@ def install_galaxy_option():
     )
 
 
+def no_cache_galaxy_option():
+    return click.option(
+        "--no_cache_galaxy",
+        is_flag=True,
+        help=("Skip caching of downloaded Galaxy obtained with "
+              "--install_galaxy. Not caching this results in faster "
+              "downloads (no git) - so is better on throw away instances such "
+              "with TravisCI. ")
+    )
+
+
 def brew_option():
     return click.option(
         '--brew',
diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh
@@ -5,4 +5,6 @@ virtualenv planemo-venv
 pip install planemo
 planemo travis_before_install
  . ${TRAVIS_BUILD_DIR}/.travis/env.sh # source environment created by planemo
+# TODO: Add --no_cache_galaxy once planemo 0.7.0 is available
+# on PyPI.
 planemo test --install_galaxy ${TRAVIS_BUILD_DIR}
diff --git a/tests/test_init_and_test.py b/tests/test_init_and_test.py
@@ -7,5 +7,10 @@ def test_init_and_test(self):
         with self._isolate():
             init_cmd = ["project_init", "--template", "demo", "basic"]
             self._check_exit_code(init_cmd)
-            test_cmd = ["test", "--install_galaxy", "basic/cat.xml"]
+            test_cmd = [
+                "test",
+                "--no_cache_galaxy",
+                "--install_galaxy",
+                "basic/cat.xml"
+            ]
             self._check_exit_code(test_cmd)

Original file line number	Diff line number	Diff line change
`@@ -84,6 +84,7 @@`
`84`	`84`	`)`
`85`	`85`	`@options.galaxy_root_option()`
`86`	`86`	`@options.install_galaxy_option()`
	`87`	`+@options.no_cache_galaxy_option()`
`87`	`88`	`@options.no_cleanup_option()`
`88`	`89`	`@options.test_data_option()`
`89`	`90`	`@options.tool_data_table_option()`