From 7e8bee8830d5d82e222b350d4bc240d1206e5af3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 30 Sep 2011 00:51:46 +0000 Subject: [PATCH 001/109] Deleting directories from SVN that should have been ignored. git-svn-id: https://svn.msi.umn.edu/tint/trunk@427 29e1c259-b37a-0410-ae23-5fb2df606144 --- galaxy.xsd | 216 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 galaxy.xsd diff --git a/galaxy.xsd b/galaxy.xsd new file mode 100644 index 000000000000..3d2915bef576 --- /dev/null +++ b/galaxy.xsd @@ -0,0 +1,216 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 39fb238e666bb404b140b9ec37f1caeeb8a2637f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 21 Nov 2011 20:25:22 -0600 Subject: [PATCH 002/109] Work on galaxy support, implemented repeat blocks in the UI. --- galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 3d2915bef576..9c251988bb55 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -104,10 +104,10 @@ - - + + From a718608e838f796bddb159b4e303194db52b3085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fr=C3=A9d=C3=A9ric?= Date: Wed, 20 Feb 2013 19:56:21 +0100 Subject: [PATCH 003/109] Add basic ReadMe --- README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000000..bcd120a1f86b --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +Galaxy-XSD +========== + +A Galaxy XML tool wrapper XSD From 6e689fa77eef72020f62e00149bddfae825e299f Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 20:12:15 +0100 Subject: [PATCH 004/109] Remove custom namespaces --- galaxy.xsd | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 9c251988bb55..294b9157e548 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1,9 +1,7 @@ + xmlns:xs="http://www.w3.org/2001/XMLSchema" + elementFormDefault="qualified" attributeFormDefault="unqualified"> From e1fc626fc1d6f2e49519784bea217ea3022d4073 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 20:14:26 +0100 Subject: [PATCH 005/109] Change tool element to Element order is not enforced by Galaxy. --- galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 294b9157e548..d74545e24f7e 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -5,7 +5,7 @@ - + @@ -13,7 +13,7 @@ - + From 025a22b0e7bbb8f9fc89dac33c8e18fdfc49b36f Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 20:51:34 +0100 Subject: [PATCH 006/109] Add tag set This includes: - - -Enumeration RequirementType --- galaxy.xsd | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index d74545e24f7e..f0848ad6e5c1 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -6,6 +6,7 @@ + @@ -21,6 +22,21 @@ + + + + + + + + + + + + + + + @@ -211,4 +227,12 @@ + + + + + + + + From 08f1faad44fde20cf0748d7c12785300d67942ec Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 20:55:25 +0100 Subject: [PATCH 007/109] Complete tag set Add attributes min and max to Repeat element --- galaxy.xsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index f0848ad6e5c1..e9b503c501e7 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -110,6 +110,8 @@ + + From 9a9ba49caab8c63cff05153948160acccd80f912 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 21:08:40 +0100 Subject: [PATCH 008/109] Add tag to The tag can contain to dynamically create a list of options. --- galaxy.xsd | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index e9b503c501e7..0c32b749f4e4 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -120,7 +120,10 @@ - + + + + @@ -172,6 +175,15 @@ + + + + + + + + + From 6b828d2b29b1612c82bcd7348625318628712aaf Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 21:13:01 +0100 Subject: [PATCH 009/109] Add - - - - - - - - + + + + + + + + From 91a166b7f5f42ee60495da20443b73d4bda1a502 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 21:36:13 +0100 Subject: [PATCH 013/109] Add tag Add the tag (which serves for splitting job) and the datatype for the 'method' attribute. --- galaxy.xsd | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 11ea0e203c41..d336a774e68c 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -8,6 +8,7 @@ + @@ -37,6 +38,10 @@ + + + + @@ -261,6 +266,13 @@ + + + + + + + From f9c5cf28ef2f60867a81dd8b6da3454e46d2936a Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 21:38:59 +0100 Subject: [PATCH 014/109] Add tag set to Add tag set, to handle tool messages on standard output and standard error. - Add tag - Add and sub elements - Add datatypes for 'range', 'level' and 'source' attributes --- galaxy.xsd | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index d336a774e68c..b13aa768cb54 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -14,6 +14,7 @@ + @@ -238,6 +239,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -295,4 +316,26 @@ + + + + + + + + + + + + + + + + + + + + + + From 6e1d7e40a333d717a7c877c18eb94048dd99a7f0 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 21:46:04 +0100 Subject: [PATCH 015/109] Change type for requirement version Version information can be too complex for xs:decimal (eg "0.12.7"). Change for xs:string. --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index b13aa768cb54..983f51d0ad09 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -34,7 +34,7 @@ - + From 642e88857caa09d968f4e5c740dc44475a64e288 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 22:10:49 +0100 Subject: [PATCH 016/109] Add column, filter and validator tags --- galaxy.xsd | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 983f51d0ad09..8f0d34f3d3dd 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -190,14 +190,43 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -316,6 +345,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1787f1e89b4a311d9bae2c60effc851347da50e2 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Wed, 20 Feb 2013 22:24:57 +0100 Subject: [PATCH 017/109] Add ftype attribute to param tag in tests --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 8f0d34f3d3dd..d39266fc8b6f 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -62,6 +62,7 @@ + From 6181d24704fd23b3370a961ba3fb573d0100f099 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:38:41 +0100 Subject: [PATCH 018/109] Add tag set --- galaxy.xsd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index d39266fc8b6f..2e62d643ac0b 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -16,6 +16,7 @@ + @@ -43,6 +44,10 @@ + + + + From 0da4d3c85361954a82fb8a0aa4c0f6bd742b9033 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:39:05 +0100 Subject: [PATCH 019/109] Remove required use from version --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 2e62d643ac0b..5fe962015e3c 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -35,7 +35,7 @@ - + From 8164cfd3ba691dbb11fc5f623610b726ad5f2b85 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:40:00 +0100 Subject: [PATCH 020/109] Edit tag set --- galaxy.xsd | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 5fe962015e3c..9d998893426d 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -135,7 +135,7 @@ - + @@ -195,19 +195,25 @@ - + - - - - + - - - - + + + + + + + + + + + + + From bb285e4caee908cf4be30f0bf19ae21ffba20ebe Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:41:01 +0100 Subject: [PATCH 021/109] Add to --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 9d998893426d..49dfb6f89e05 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -137,6 +137,7 @@ + From 02aa90c4a477bb595540914df08ef9402275d84d Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:41:34 +0100 Subject: [PATCH 022/109] Add 'split' attribute to --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 49dfb6f89e05..6b735a99c29d 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -229,6 +229,7 @@ + From e200148feef9ff553820272d7893b387e6e4723d Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 12:41:50 +0100 Subject: [PATCH 023/109] Add tag set --- galaxy.xsd | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 6b735a99c29d..b621d886b01f 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -251,6 +251,7 @@ + @@ -261,6 +262,25 @@ + + + + + + + + + + + + + + + + + + + @@ -386,6 +406,13 @@ + + + + + + + From 1c9513d6cb64315e742eab7befec7536dee766c8 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:18:49 +0100 Subject: [PATCH 024/109] Extend CheckedType to accept unspecified values According to the documentation, CheckedType can either be 'yes', 'true' or 'on' ; but many tools use '[Ff]alse' which seems to be accepted/ignored by Galaxy. --- galaxy.xsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index b621d886b01f..a0425029f1ea 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -376,6 +376,8 @@ + + From b35b9fd52c78a891555660d7512e617b03d36cc0 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:20:49 +0100 Subject: [PATCH 025/109] Add PermissiveBoolean datatype can either be 'true', 'false', '1' or 'O' ; but many Galaxy tools use 'True' and 'False' (capitalized), which appears to be ignored/accepted by Galaxy. This PermissiveBoolean type is meant to be used in place of xs:boolean when needed. --- galaxy.xsd | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index a0425029f1ea..c59268e7a007 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -437,4 +437,15 @@ - + + + + + + + + + + + + \ No newline at end of file From dd0ba4293dc03db422a7b33e974c3b7554c287e9 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:23:01 +0100 Subject: [PATCH 026/109] Add attributes 'ftype' and 'sort' to test These are undocumented attributes. --- galaxy.xsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index c59268e7a007..e56e97490340 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -76,6 +76,8 @@ + + From 4578e0a0f957ef7f851257b7bb605764712c5a44 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:25:56 +0100 Subject: [PATCH 027/109] Add (undocumented) attributes to Add attributes 'compare', 'keep' and 'value' to tag (though these attributes are not documented). Add CompareType for possible 'compare' values. --- galaxy.xsd | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index e56e97490340..75f37fc3ebd1 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -241,6 +241,9 @@ + + + @@ -417,6 +420,12 @@ + + + + + + From 4bdfd455c58d00c5a28fdf1e6fc80d70648cb57a Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:27:58 +0100 Subject: [PATCH 028/109] Edit filter tag for output tag --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 75f37fc3ebd1..c034aed60762 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -255,7 +255,7 @@ - + From 566e279172c98e417c7fca9cf04bda2e3abd2da5 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:28:13 +0100 Subject: [PATCH 029/109] Rewrite handling --- galaxy.xsd | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index c034aed60762..74a021d3aa0a 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -269,23 +269,54 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -420,6 +451,13 @@ + + + + + + + From 1cda2e4439f28229d000f30120ecf69709606b6f Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:41:30 +0100 Subject: [PATCH 030/109] Add (undocumented) 'dbkey' attribute to TestParam --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 74a021d3aa0a..54968268603c 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -68,6 +68,7 @@ + From 081c39e55161e1d1babebed1d159f2415a8e0adb Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:42:16 +0100 Subject: [PATCH 031/109] Add 'from_file' value to ActionsOptionType --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 54968268603c..84b2f596cda8 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -456,6 +456,7 @@ + From b466de7e0a5e019d1a4264f2982f8c2012715241 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:44:46 +0100 Subject: [PATCH 032/109] Use PermissiveBoolean for ParamOption 'selected' --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 84b2f596cda8..4c6d4dd3ea8b 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -194,7 +194,7 @@ - + From 92b1076261ee0403ff8fb6f8e2601ba5149e9807 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:57:12 +0100 Subject: [PATCH 033/109] Add attributes tool_type and URL_method to And corresponding enumeration Datatypes. --- galaxy.xsd | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 4c6d4dd3ea8b..17198297f928 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -22,6 +22,8 @@ + + @@ -387,6 +389,19 @@ + + + + + + + + + + + + + From e8f236a52e5c4081ef9a228e15ffefb50185db7f Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:58:07 +0100 Subject: [PATCH 034/109] use PermissiveBoolean on two attributes of --- galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 17198297f928..69a894769b0e 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -156,8 +156,8 @@ - - + + From 7d7b5a8d3dfc604335626f0abf806377dc2b630c Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 15:58:37 +0100 Subject: [PATCH 035/109] Add (undocumented) value 'no' to CheckedType --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index 69a894769b0e..598a262177e5 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -430,6 +430,7 @@ + From e19f1fb99a982cedebc74a3b42f0c376e19d46bc Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 16:38:35 +0100 Subject: [PATCH 036/109] Add (undocumented) to See /lib/galaxy/tools/__init__.py:1069 for usage --- galaxy.xsd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 598a262177e5..563d964a88f2 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -17,6 +17,7 @@ + @@ -50,6 +51,10 @@ + + + + From 836aff4b3f70d6073bbd5f4d78d4b6688aed7baa Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 17:16:46 +0100 Subject: [PATCH 037/109] Add (undocumented) attributes to Add attributes 'action', 'check_values', 'method' and 'target' to . Add relevant Target datatype. --- galaxy.xsd | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 563d964a88f2..41f1c5424fca 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -92,6 +92,10 @@ + + + + @@ -407,6 +411,13 @@ + + + + + + + From c4769e701302625031db4a7730e572b32c019ead Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 17:20:13 +0100 Subject: [PATCH 038/109] Add tag --- galaxy.xsd | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 41f1c5424fca..dc46ff0c6da6 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -152,6 +152,7 @@ + @@ -244,7 +245,46 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 5419b1a4bce60a4459b59711acbf9c2652d5722d Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 17:53:43 +0100 Subject: [PATCH 039/109] Add (undocumented) attribute metadata_name to --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index dc46ff0c6da6..616cc38d19b9 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -173,6 +173,7 @@ + From 79cf1c57d8b6a1add066bf61ec18936bf1a8f6ee Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 17:54:18 +0100 Subject: [PATCH 040/109] Add (undocumented) attributes to Add 'options_filter_attribute' attribute (see lib/galaxy/tools/parameters/basic.py) Add 'transform_lines' attribute (see lib/galaxy/tools/parameters/dynamic_options.py) transform_lines and --- galaxy.xsd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 616cc38d19b9..ad6dd7584620 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -220,6 +220,8 @@ + + From 11b013f402b90c742789df87b6810e156a173171 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 17:59:53 +0100 Subject: [PATCH 041/109] Add (undocumented) 'ref_attribute' to Add attribute 'ref_attribute' to (see lib/galaxy/tools/parameters/dynamic_options.py:153) --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index ad6dd7584620..c4b4a97fa840 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -299,6 +299,7 @@ + From 65903327fa21536d44822ef94a9e75b2d29cf11d Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 18:04:01 +0100 Subject: [PATCH 042/109] Add (undocumented) 'startswith' to Add attribute 'startswith' to tag (see lib/galaxy/tools/parameters/dynamic_options.py:416) --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index c4b4a97fa840..f69aeb95c889 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -222,6 +222,7 @@ + From d12766e260ea8d1e6ebe61806cb7e2b3eb55b886 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 18:13:35 +0100 Subject: [PATCH 043/109] Add (undocumented) attribute 'display' to See lib/galaxy/tools/__init__.py:1531 --- galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy.xsd b/galaxy.xsd index f69aeb95c889..a3b6173eaa06 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -103,6 +103,7 @@ + From 091b3b85687eeb8ef5becdf30ff81211b12f6f45 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 18:55:58 +0100 Subject: [PATCH 044/109] Fix value baseurl in 'type' attribute --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index a3b6173eaa06..b45e0c9a23e2 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -189,7 +189,7 @@ - + From 395780d3d143982204c1f9ccf188df1deff51d55 Mon Sep 17 00:00:00 2001 From: JeanFred Date: Thu, 21 Feb 2013 18:56:50 +0100 Subject: [PATCH 045/109] Fix type of min and max in --- galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index b45e0c9a23e2..9ce9aff742c1 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -247,8 +247,8 @@ - - + + From 2fdf2267b841d2701db424f9d2f458d8fbc2309b Mon Sep 17 00:00:00 2001 From: Pierre Lindenbaum Date: Tue, 10 Feb 2015 12:46:48 +0100 Subject: [PATCH 046/109] added doc and JXB bindings --- .gitignore | 2 + Makefile | 15 + README.md | 20 +- citation.xsd | 20 + citations.xsd | 15 + galaxy.jxb | 22 + galaxy.xsd | 1296 ++++++++++++++++++++++++++++++++++++------------- test.urls | 138 ++++++ 8 files changed, 1201 insertions(+), 327 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 citation.xsd create mode 100644 citations.xsd create mode 100644 galaxy.jxb create mode 100644 test.urls diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..472ea98b6c6a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +generated +*~ diff --git a/Makefile b/Makefile new file mode 100644 index 000000000000..f1d2326b5efa --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +SHELL=/bin/bash +.PHONY: all clean tests + +all: tests + +tests: galaxy.xsd galaxy.jxb generated/ObjectFactory.java + sed "s%^%https://bitbucket.org/galaxy/galaxy-central/raw/c3eefbdaaa1ab242a1c81b65482ef2fbe943a390/%" test.urls |\ + while read L; do echo "#$${L}" && curl -ksL "$${L}" | xmllint --noout --schema $< - ; done + + +generated/ObjectFactory.java : galaxy.jxb galaxy.xsd + $(if $(realpath ${JAVA_HOME}/bin/xjc),xjc -b $^) + +clean: + rm -rf generated diff --git a/README.md b/README.md index bcd120a1f86b..ff3437e9c0d1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,22 @@ Galaxy-XSD ========== -A Galaxy XML tool wrapper XSD +A Galaxy XML tool wrapper __XML schema definition__ (__XSD__) + + +# History + +* 2013 : initial xsd by Jean-Fred +* 2015 : added doc, tests, Java-XML binding file (jxb) for java xml compiler (xjc) ( https://docs.oracle.com/cd/E19575-01/819-3669/bnbal/index.html ) + + +# Authors + +* Jean-Frédéric @JeanFred +* Pierre Lindenbaum @yokofakun + +# See also: + +* Galaxy https://usegalaxy.org/ +* Galaxy Tool XML File https://wiki.galaxyproject.org/Admin/Tools/ToolConfigSyntax + diff --git a/citation.xsd b/citation.xsd new file mode 100644 index 000000000000..54916524d060 --- /dev/null +++ b/citation.xsd @@ -0,0 +1,20 @@ + + + + + + + Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + type of citation + + + + + + + diff --git a/citations.xsd b/citations.xsd new file mode 100644 index 000000000000..3db22e7c9506 --- /dev/null +++ b/citations.xsd @@ -0,0 +1,15 @@ + + + + + + + Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + + + diff --git a/galaxy.jxb b/galaxy.jxb new file mode 100644 index 000000000000..29eac0bfabc6 --- /dev/null +++ b/galaxy.jxb @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + diff --git a/galaxy.xsd b/galaxy.xsd index 9ce9aff742c1..6ad7248bb41e 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1,572 +1,1217 @@ - - + elementFormDefault="qualified" + attributeFormDefault="unqualified" + > + + Galaxy Schema + A Galaxy XML tool wrapper + + + + + + + The outer-most tag set + - - - - - - - - - - - - + + + + The value is displayed in the tool menu immediately following the hyperlink for the tool + + + + + + Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. + + + + + + + + + + + Documentation for help + + + + + - - - - - - + + + Must be unique across all tools; should be lowercase and contain only letters, numbers, and underscores. It allows for tool versioning and metrics of the number of times a tool is used, among other things. + + + + + This string is what is displayed as a hyperlink in the tool menu + + + + + This string defaults to "1.0.0' if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. + + + + + Allows for tools to be loaded upon server startup, but not displayed in the tool menu + + + + + Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. + + + + + Only if "tool_type" attribute value is "data_source" - defines the HTTP request method to use when communicating with an external data source application ( the default is "get" ). + + - + + + This is a container tag set for the 'requirement' tag set described below. + - + - + + + This tag set is contained within the 'requirements' tag set. Third party programs or modules that the tool depends upon (and which are not distributed with Galaxy) are included in this tag set. The intention is that when Galaxy starts it can check whether the required programs or modules are available, and if not this tool will not be loaded. + - - + + + This value defines the which type of the 3rd party module required by this tool + + + + + Required for package type requirements + + - + + - - - + + Documentation for Parallelism + + + + Documentation for method + + + - - - + + Documentation for Code + + + + Documentation for file + + + - - - + + Documentation for UIhints + + + + Documentation for minwidth + + + + + Documentation for Tests + - + - + + Documentation for Test + - - + + - + + Documentation for TestParam + - + + + Documentation for embeddedValue + + - - - - + + + Documentation for name + + + + + Documentation for value + + + + + Documentation for ftype + + + + + Documentation for dbkey + + - + + Documentation for TestOutput + - + + + Documentation for embeddedValue + + - - - - + + + Documentation for name + + + + + Documentation for file + + + + + Documentation for ftype + + + + + Documentation for sort + + - + + + + Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. + - + - - - - + + + Documentation for action + + + + + Documentation for check_values + + + + + Documentation for method + + + + + Documentation for target + + - + + - - - - + + + + + + Documentation for display + + - - - + + Documentation for InputType + + + + + Name for this element + + - + + + + This is a container for conditional parameters in the tool ( must contain 'when' tag sets ) - the command line is wrapped in an if-else statement. + - - + + - + + + + + Documentation for ConditionalWhen + - + - + + + Documentation for value + + + + + + This is a container for any tag sets that can be contained within the 'inputs' tag set. When this is used, the tool will allow the user to add any number of additional sets of the contained parameters ( an "Add new 'title'" button will be displayed on the tool form ). + - + - - - + + + The title of the repeat section, which will be displayed on the tool form + + + + + The minimum number of repeat units + + + + + The maximum number of repeat units + + + + + The default number of repeat units + + - + + + + + Documentation for Param + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + Documentation for label + + + + + + + + + + + + Documentation for type + + + + + Documentation for label + + + + + Documentation for help + + + + + Documentation for value + + + + + Documentation for optional + + + + + Documentation for min + + + + + Documentation for max + + + + + Documentation for format + + + + + Documentation for data_ref + + + + + Documentation for accept_default + + + + + Documentation for force_select + + + + + Documentation for display + + + + + Documentation for multiple + + + + + Documentation for numerical + + + + + Documentation for hierarchy + + + + + Documentation for checked + + + + + Documentation for truevalue + + + + + Documentation for falsevalue + + + + + Documentation for size + + + + + Documentation for metadata_name + + - + + Documentation for ParamType + - - - - - - - - - - - - + + + + + + + + + + + + - + + + + This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). + - + + + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. + + - + + + + Documentation for ParamOption + - - + + + Documentation for value + + + + + Documentation for selected + + - + + + + Documentation for Options + - + - - - - - - - - - + + + Documentation for from_dataset + + + + + Documentation for from_file + + + + + Documentation for from_data_table + + + + + Documentation for from_parameter + + + + + Documentation for options_filter_attribute + + + + + Documentation for transform_lines + + + + + Documentation for startswith + + + - - - - + + + + + + Documentation for file + + - - - - - + + Documentation for Column + + + + Documentation for name + + + + + Documentation for index + + + - - - - - - - - - - - + + Documentation for Validator + + + + Documentation for type + + + + + Documentation for message + + + + + Documentation for filename + + + + + Documentation for metadata_name + + + + + Documentation for metadata_column + + + + + Documentation for line_startswith + + + + + Documentation for min + + + + + Documentation for max + + + + + Documentation for split + + + + + Documentation for Sanitizer + - + - - + + + Documentation for sanitize + + + + + Documentation for invalid_char + + - + + Documentation for Valid + - + - + + + Documentation for initial + + - - - + + Documentation for ValidElementType + + + + Documentation for preset + + + + + Documentation for value + + - + + Documentation for Add + - + - + + Documentation for Remove + - + - - - + + - - - - - - - - - - - - - + + Documentation for Filter + + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for column + + + + + Documentation for ref + + + + + Documentation for key + + + + + Documentation for separator + + + + + Documentation for compare + + + + + Documentation for keep + + + + + Documentation for value + + + + + Documentation for ref_attribute + + + + + Documentation for Outputs + - + - + + Documentation for Data + - - - + + + + Documentation for filter + + + - - - - - - - - - + + + Documentation for name + + + + + Documentation for format + + + + + Documentation for format_source + + + + + Documentation for metadata_source + + + + + Documentation for label + + + + + Documentation for from_work_dir + + + + + Documentation for hidden + + + + + Documentation for Actions + - + - - - + + - + + Documentation for Action + - + - - - + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for default + + - - + + + Documentation for ActionsOption + - + - - - - - + + + Documentation for type + + + + + Documentation for name + + + + + Documentation for column + + + + + Documentation for offset + + + + + Documentation for param_attribute + + - + + Documentation for ActionsConditional + - - - + + + - + + Documentation for ActionsConditionalWhen + - + - + + + Documentation for value + + - + + Documentation for ConfigFiles + - + - + + Documentation for ConfigFile + - - - + + + Documentation for name + + + + + Documentation for type + + - + + + + Galaxy currently supports using regular expressions to scan stdout and stderr, and it also allows exit codes to be scanned for ranges. The 'stdio' tag has two subtags, 'regex' and 'exit_code', to define regular expressions and exit code processing, respectively. + - - + + - + + - - - - - + + Documentation for ExitCode + + + + Documentation for range + + + + + Documentation for level + + + + + Documentation for description + + + + + - - - - + + A regular expression defines a pattern of characters + + + + This tells whether the regular expression should be matched against stdout, stderr, or both. If this attribute is missing or is incorrect, then both stdout and stderr will be checked. The source can be one of the follwing values: + + + + + This is the regular expression that will be used to match against stdout and/or stderr. + + + + + This works very similarly to the 'exit_code' tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. + + + + + an optional description of the regular expression that has matched. + + - + + + + Documentation for ConfigFileType + - - + + - + + + Documentation for ChangeFormat + - + - - - - - + + Documentation for WhenData + + + + + Documentation for input + + + + + Documentation for value + + + + + Documentation for format + + - + + Documentation for RequirementType + - - - + + + - + + Documentation for ToolTypeType + - + - + + Documentation for URLmethodType + - - + + - + + Documentation for TargetType + - - + + - + + Documentation for MethodType + - - + + - + + Documentation for DisplayType + - - + + - + + Documentation for HierarchyType + - - + + - + + Documentation for CheckedType + - - - - - - + + + + + + - + + Documentation for ValidatorType + - - - - - - - - - - + + + + + + + + + + - + + Documentation for FilterType + - - - - - - - + + + + + + + - + + Documentation for ActionType + - - + + - + + Documentation for ActionsOptionType + - - - + + + - - + + + Documentation for CompareType + - + - + + Documentation for LevelType + - - - + + + - + + Documentation for RangeType + - + + Documentation for SourceType + - - - + + + - - + + + Documentation for PermissiveBoolean + @@ -576,5 +1221,4 @@ - - \ No newline at end of file + diff --git a/test.urls b/test.urls new file mode 100644 index 000000000000..56415dac3c3c --- /dev/null +++ b/test.urls @@ -0,0 +1,138 @@ +tools/sr_assembly/velvetg.xml +tools/sr_assembly/velveth.xml +tools/data_source/microbial_import.xml +tools/data_source/wormbase_test.xml +tools/data_source/ucsc_archaea.xml +tools/data_source/worm_modencode.xml +tools/data_source/bed_convert.xml +tools/data_source/gramene_mart.xml +tools/data_source/ebi_sra.xml +tools/data_source/modmine.xml +tools/data_source/import.xml +tools/data_source/echo.xml +tools/data_source/biomart.xml +tools/data_source/ucsc_tablebrowser_test.xml +tools/data_source/flymine_test.xml +tools/data_source/ucsc_proxy.xml +tools/data_source/biomart_test.xml +tools/data_source/access_libraries.xml +tools/data_source/ucsc_tablebrowser.xml +tools/data_source/hbvar.xml +tools/data_source/wormbase.xml +tools/data_source/flymine.xml +tools/data_source/ucsc_tablebrowser_archaea.xml +tools/data_source/metabolicmine.xml +tools/data_source/eupathdb.xml +tools/data_source/yeastmine.xml +tools/data_source/fly_modencode.xml +tools/data_source/upload.xml +tools/data_source/mousemine.xml +tools/data_source/cbi_rice_mart.xml +tools/data_source/hapmapmart.xml +tools/data_source/ucsc_testproxy.xml +tools/data_source/ratmine.xml +tools/data_source/genbank.xml +tools/visualization/LAJ.xml +tools/stats/aggregate_binned_scores_in_intervals.xml +tools/stats/filtering.xml +tools/stats/grouping.xml +tools/stats/gsummary.xml +tools/ngs_simulation/ngs_simulation.xml +tools/maf/maf_by_block_number.xml +tools/maf/maf_to_interval.xml +tools/maf/maf_to_bed.xml +tools/maf/maf_limit_size.xml +tools/maf/vcf_to_maf_customtrack.xml +tools/maf/maf_limit_to_species.xml +tools/maf/maf_filter.xml +tools/maf/maf_split_by_species.xml +tools/maf/maf_to_fasta.xml +tools/maf/genebed_maf_to_fasta.xml +tools/maf/interval2maf_pairwise.xml +tools/maf/interval2maf.xml +tools/maf/maf_stats.xml +tools/maf/maf_thread_for_species.xml +tools/maf/macros.xml +tools/maf/interval_maf_to_merged_fasta.xml +tools/maf/maf_reverse_complement.xml +tools/metag_tools/shrimp_wrapper.xml +tools/metag_tools/blat_wrapper.xml +tools/metag_tools/shrimp_color_wrapper.xml +tools/genomespace/genomespace_exporter.xml +tools/genomespace/genomespace_file_browser_dev.xml +tools/genomespace/genomespace_importer.xml +tools/genomespace/genomespace_file_browser_prod.xml +tools/genomespace/genomespace_file_browser_test.xml +tools/plotting/bar_chart.xml +tools/plotting/boxplot.xml +tools/solid_tools/maq_cs_wrapper.xml +tools/solid_tools/solid_qual_stats.xml +tools/solid_tools/solid_qual_boxplot.xml +tools/sr_mapping/srma_wrapper.xml +tools/sr_mapping/PerM.xml +tools/sr_mapping/bfast_wrapper.xml +tools/sr_mapping/fastq_statistics.xml +tools/sr_mapping/mosaik.xml +tools/extract/extract_genomic_dna.xml +tools/extract/liftOver_wrapper.xml +tools/validation/fix_errors.xml +tools/meme/meme.xml +tools/meme/fimo.xml +tools/next_gen_conversion/solid_to_fastq.xml +tools/next_gen_conversion/fastq_conversions.xml +tools/next_gen_conversion/solid2fastq.xml +tools/next_gen_conversion/fastq_gen_conv.xml +tools/evolution/add_scores.xml +tools/evolution/codingSnps.xml +tools/phenotype_association/master2pg.xml +tools/phenotype_association/pass.xml +tools/phenotype_association/lps.xml +tools/phenotype_association/ldtools.xml +tools/phenotype_association/master2gd_snp.xml +tools/phenotype_association/sift.xml +tools/phenotype_association/linkToDavid.xml +tools/phenotype_association/beam.xml +tools/phenotype_association/linkToGProfile.xml +tools/phenotype_association/gpass.xml +tools/filters/CreateInterval.xml +tools/filters/convert_characters.xml +tools/filters/bed2gff.xml +tools/filters/mergeCols.xml +tools/filters/sorter.xml +tools/filters/trimmer.xml +tools/filters/uniq.xml +tools/filters/wig_to_bigwig.xml +tools/filters/ucsc_gene_bed_to_exon_bed.xml +tools/filters/gff2bed.xml +tools/filters/fileGrep.xml +tools/filters/randomlines.xml +tools/filters/axt_to_fasta.xml +tools/filters/cutWrapper.xml +tools/filters/gtf2bedgraph.xml +tools/filters/axt_to_lav.xml +tools/filters/ucsc_gene_table_to_intervals.xml +tools/filters/pasteWrapper.xml +tools/filters/gff/extract_GFF_Features.xml +tools/filters/gff/gtf_filter_by_attribute_values_list.xml +tools/filters/gff/gff_filter_by_feature_count.xml +tools/filters/gff/gff_filter_by_attribute.xml +tools/filters/grep.xml +tools/filters/fixedValueColumn.xml +tools/filters/catWrapper.xml +tools/filters/condense_characters.xml +tools/filters/headWrapper.xml +tools/filters/wc_gnu.xml +tools/filters/axt_to_concat_fasta.xml +tools/filters/commWrapper.xml +tools/filters/tailWrapper.xml +tools/filters/bed_to_bigbed.xml +tools/filters/lav_to_bed.xml +tools/filters/joiner.xml +tools/filters/changeCase.xml +tools/filters/compare.xml +tools/filters/wiggle_to_simple.xml +tools/filters/secure_hash_message_digest.xml +tools/filters/sff_extractor.xml +tools/filters/remove_beginning.xml +tools/filters/ucsc_gene_bed_to_intron_bed.xml +tools/filters/joiner2.xml From 2cecbe1922a9b5d75f34d42bedcd025a83fbae09 Mon Sep 17 00:00:00 2001 From: Pierre Lindenbaum Date: Tue, 10 Feb 2015 15:21:55 +0100 Subject: [PATCH 047/109] updated README --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ff3437e9c0d1..78829ac007fb 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,23 @@ Galaxy-XSD A Galaxy XML tool wrapper __XML schema definition__ (__XSD__) + # History +* Feb-2015 : Pierre Lindenbaum added doc, tests, Java-XML binding file (jxb) for java xml compiler (xjc) ( https://docs.oracle.com/cd/E19575-01/819-3669/bnbal/index.html ) * 2013 : initial xsd by Jean-Fred -* 2015 : added doc, tests, Java-XML binding file (jxb) for java xml compiler (xjc) ( https://docs.oracle.com/cd/E19575-01/819-3669/bnbal/index.html ) + +# Validating a `tool.xml` + +```bash +$ xmllint --noout --schema galaxy.xsd tool.xml +``` + +# Creating java code + +```bash +$ ${JAVA_HOME}/bin/xjc -b galaxy.jxb galaxy.xsd +``` # Authors From f692d3a5ae244ce638be596723f369dade3ed501 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Fri, 13 Feb 2015 19:16:35 +0000 Subject: [PATCH 048/109] Add Continuous integration with Travis for XSD linting At the very least these XSD files should be XML valid. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..ea4596b9f48e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +install: + - apt-get install libxml2-utils +script: + - xmllint --noout *.xsd From cdbe79b4b657609724081c991466b0be13460d36 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Fri, 13 Feb 2015 19:21:16 +0000 Subject: [PATCH 049/109] Fix dependency installation for Travis Build was failing with $ apt-get install libxml2-utils E: Could not open lock file /var/lib/dpkg/lock - open (13: Permission denied) E: Unable to lock the administration directory (/var/lib/dpkg/), are you root? APT dependencies must be installed as sudoer. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea4596b9f48e..f1462b08227b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ install: - - apt-get install libxml2-utils + - sudo apt-get install libxml2-utils script: - xmllint --noout *.xsd From 00a560ce1dbb093989b36f628abf08bf5ad3ee91 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Fri, 13 Feb 2015 19:28:28 +0000 Subject: [PATCH 050/109] Add Travis badge to ReadMe --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 78829ac007fb..d9595dea2de2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ Galaxy-XSD ========== +[![Build Status](https://travis-ci.org/JeanFred/Galaxy-XSD.svg)](http://travis-ci.org/JeanFred/Galaxy-XSD) A Galaxy XML tool wrapper __XML schema definition__ (__XSD__) From 3315e9a8c424281df01e7521a9c9715447356059 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Mon, 16 Feb 2015 14:58:21 +0000 Subject: [PATCH 051/109] Add Shell test script Following the idea of @lindenb in the Makefile. This tests all the given URLs in test.urls and outputs a compliant TAP. --- test.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 test.sh diff --git a/test.sh b/test.sh new file mode 100644 index 000000000000..a9bec6e31bb0 --- /dev/null +++ b/test.sh @@ -0,0 +1,18 @@ +#! /bin/bash +count=`wc -l test.urls | cut -f1 -d' '` +root_url="https://bitbucket.org/galaxy/galaxy-central/raw/c3eefbdaaa1ab242a1c81b65482ef2fbe943a390/" +echo "1..$count" +count=0 +while read p; do + count=$((count+1)) + url="$root_url""$p" + result=`curl -ksL "$url" | xmllint --nowarning --noout --schema galaxy.xsd - 2> err.tmp` + if [ $? -eq 0 ] + then + echo "ok $count $url"; + else + echo "not ok $count $url"; + cat err.tmp | sed 's/^/ /' + fi +done Date: Sun, 15 Feb 2015 10:54:21 -0500 Subject: [PATCH 052/109] Add me (@jmchilton) to authors list. My name is on the earliest commits (thanks for preserving that). --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d9595dea2de2..5cc6ddd83d72 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ $ ${JAVA_HOME}/bin/xjc -b galaxy.jxb galaxy.xsd * Jean-Frédéric @JeanFred * Pierre Lindenbaum @yokofakun +* John Chilton @jmchilton # See also: From 1b7f2a54f6c2b12d9340fe4c8c3e8de0f0f68afb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 15 Feb 2015 11:07:40 -0500 Subject: [PATCH 053/109] Fix CitationType attribute. Add Galaxy's test tool for citations to test.urls. --- citation.xsd | 12 +++++++++++- test.urls | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/citation.xsd b/citation.xsd index 54916524d060..81c470687fb5 100644 --- a/citation.xsd +++ b/citation.xsd @@ -8,7 +8,7 @@ - + type of citation @@ -16,5 +16,15 @@ + + + + Type of citation represented. + + + + + + diff --git a/test.urls b/test.urls index 56415dac3c3c..8ccffcd0ee57 100644 --- a/test.urls +++ b/test.urls @@ -1,3 +1,4 @@ +test/functional/tools/bibtex.xml tools/sr_assembly/velvetg.xml tools/sr_assembly/velveth.xml tools/data_source/microbial_import.xml From 057c36997bd5c4c75a85acd5a9346f8c28eee015 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 15 Feb 2015 11:08:26 -0500 Subject: [PATCH 054/109] Update XSD for container tags. Add Galaxy test tool for Docker container tags to test.urls. --- galaxy.xsd | 27 +++++++++++++++++++++++++-- test.urls | 1 + 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 6ad7248bb41e..27bf7ac875c3 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -84,6 +84,7 @@ + @@ -106,8 +107,22 @@ - - + + + This tag set is contained within the 'requirements' tag set. Galaxy can be configured to run tools within Docker (https://www.docker.com/) containers - this tag allows the tool to suggest possible valid Docker containers for this tool. + + + + + + + This value describes the type of container that the tool may be executed in and currently must be 'docker'. + + + + + + Documentation for Parallelism @@ -1056,6 +1071,14 @@ + + + Type of container for tool execution. + + + + + Documentation for ToolTypeType diff --git a/test.urls b/test.urls index 8ccffcd0ee57..e0bb205f7f7c 100644 --- a/test.urls +++ b/test.urls @@ -1,4 +1,5 @@ test/functional/tools/bibtex.xml +test/functional/tools/catDocker.xml tools/sr_assembly/velvetg.xml tools/sr_assembly/velveth.xml tools/data_source/microbial_import.xml From bdf36556fbf4c4db4c343e51d6d78c1ac2030464 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 15 Feb 2015 11:21:36 -0500 Subject: [PATCH 055/109] Drop embeddedValue for test stuff. These were an extension to the Galaxy XML syntax for TINT - which is no longer actively maintained. --- galaxy.xsd | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 27bf7ac875c3..6b929a22c7af 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -175,11 +175,6 @@ Documentation for TestParam - - - Documentation for embeddedValue - - @@ -207,11 +202,6 @@ Documentation for TestOutput - - - Documentation for embeddedValue - - From f0799b5fbb9dfe54241b702c450f258c946bf326 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 15 Feb 2015 11:34:27 -0500 Subject: [PATCH 056/109] Start work on implementing assert_contents tag in tests. Just enough to get Galaxy's simple_contructrs.xml to validate - add this to test.urls. More work should be done on these - but it should be automated derived from Galaxy somehow. --- galaxy.xsd | 24 ++++++++++++++++++++++-- test.urls | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 6b929a22c7af..0d58019be75a 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -202,6 +202,7 @@ Documentation for TestOutput + @@ -224,8 +225,27 @@ - - + + + Documentation for TestAssertion + + + + + + + + + + + + + + + Line the test output must contain. + + + Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. diff --git a/test.urls b/test.urls index e0bb205f7f7c..4fab41134cd3 100644 --- a/test.urls +++ b/test.urls @@ -1,5 +1,6 @@ test/functional/tools/bibtex.xml test/functional/tools/catDocker.xml +test/functional/tools/simple_constructs.xml tools/sr_assembly/velvetg.xml tools/sr_assembly/velveth.xml tools/data_source/microbial_import.xml From b479b7334fc6649606c5144da7c9225c82857e6b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 15 Feb 2015 11:08:42 -0500 Subject: [PATCH 057/109] Update XSD for very basic data collection usage. - Allow 'data_collection' parameters and allow them to specify a 'collection_type'. - Allow test parameters to be flat collections. - Add test/functional/tools/collection_paired_test.xml to test.urls to check all of this. Lots more TODO for recent tool collection support - output collections definitions, output collection tests, dynamic discovery tags, specifying nested collections for test inputs. --- galaxy.xsd | 20 ++++++++++++++++++++ test.urls | 1 + 2 files changed, 21 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 0d58019be75a..fbcde8349c0e 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -175,6 +175,7 @@ Documentation for TestParam + @@ -197,6 +198,19 @@ + + + Definition of a collection for test input. + + + + + + + Type of collection to create. + + + Documentation for TestOutput @@ -427,6 +441,11 @@ Documentation for format + + + Restrict valid collection types for data_collection params (e.g. list, paired, list:paired, etc...). + + Documentation for data_ref @@ -507,6 +526,7 @@ + diff --git a/test.urls b/test.urls index 4fab41134cd3..a7b967442300 100644 --- a/test.urls +++ b/test.urls @@ -1,6 +1,7 @@ test/functional/tools/bibtex.xml test/functional/tools/catDocker.xml test/functional/tools/simple_constructs.xml +test/functional/tools/collection_paired_test.xml tools/sr_assembly/velvetg.xml tools/sr_assembly/velveth.xml tools/data_source/microbial_import.xml From 49f7d51f57eb2f72e01e37fdaf6c44ffe11898de Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Mon, 16 Feb 2015 15:12:05 +0000 Subject: [PATCH 058/109] Add MIT license Per agreement of all authors. Closes #2. --- LICENSE | 22 ++++++++++++++++++++++ README.md | 7 +++++++ 2 files changed, 29 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..4c00a2d0e535 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2013 Pierre-Selim, Jean-Frédéric + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/README.md b/README.md index 5cc6ddd83d72..d7e497c6f8b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ Galaxy-XSD ========== [![Build Status](https://travis-ci.org/JeanFred/Galaxy-XSD.svg)](http://travis-ci.org/JeanFred/Galaxy-XSD) +[![License](http://img.shields.io/badge/license-MIT-orange.svg?style=flat)](http://opensource.org/licenses/MIT) A Galaxy XML tool wrapper __XML schema definition__ (__XSD__) @@ -30,6 +31,12 @@ $ ${JAVA_HOME}/bin/xjc -b galaxy.jxb galaxy.xsd * Pierre Lindenbaum @yokofakun * John Chilton @jmchilton + +# Licence + +This code is free software released under the terms of the MIT license. + + # See also: * Galaxy https://usegalaxy.org/ From 8947f29ef2596e750d2783206df547efc7ef40fd Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Mon, 16 Feb 2015 15:12:55 +0000 Subject: [PATCH 059/109] Expand history in ReadMe --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d7e497c6f8b9..60274d37b3b2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ A Galaxy XML tool wrapper __XML schema definition__ (__XSD__) # History * Feb-2015 : Pierre Lindenbaum added doc, tests, Java-XML binding file (jxb) for java xml compiler (xjc) ( https://docs.oracle.com/cd/E19575-01/819-3669/bnbal/index.html ) -* 2013 : initial xsd by Jean-Fred +* 2013 : extraction to standalone and improvements by Jean-Fred +* 2011 : Initial work by John Chilton # Validating a `tool.xml` From 5d40c7b3e2b66d91a52ae12cf48cda30d245d180 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Mon, 16 Feb 2015 15:37:30 +0000 Subject: [PATCH 060/109] Fix authors in License file Follow up on #2. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 4c00a2d0e535..bfa63c52c28e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 Pierre-Selim, Jean-Frédéric +Copyright (c) 2011-2015 John Chilton, Jean-Frédéric Berthelot, Pierre Lindenbaum Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From a4e5568af7f206c61ebf7afd9165bd1e34181d0b Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 18 Sep 2015 13:53:27 -0700 Subject: [PATCH 061/109] first try --- galaxy.xsd | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index fbcde8349c0e..7580bd115443 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -168,6 +168,7 @@ + @@ -239,6 +240,24 @@ + + + Documentation for TestOutputCollection + + + + + + + Documentation for name + + + + + Documentation for type + + + Documentation for TestAssertion @@ -250,6 +269,7 @@ + @@ -260,6 +280,28 @@ + + + + Text the test output must contain. + + + + + + + + + + Documentation for file + + + + + Documentation for name + + + Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. @@ -817,6 +859,7 @@ + @@ -868,6 +911,44 @@ + + + Documentation for Collection + + + + + + + Documentation for name + + + + + Documentation for format + + + + + Documentation for label + + + + + + Documentation for DiscoverDatasets + + + + Documentation for pattern + + + + + Documentation for directory + + + Documentation for Actions From 8f8cc890bcfaceaf4870daa2d4e22e10018119c2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 17:13:45 -0400 Subject: [PATCH 062/109] Improvements to XSD: - Implement section. - Implement and document request_parameter_translation, request_parameter and its children (value_translation and append_param). - Implement detect_errors on the command block and document defaults. --- galaxy.xsd | 227 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 222 insertions(+), 5 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 7580bd115443..c65a7ff986d6 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -14,7 +14,21 @@ - The outer-most tag set + +``` + ]]> @@ -337,6 +351,7 @@ + Documentation for display @@ -421,8 +436,32 @@ - - + + + + + TODOWIKIMOVE + + + + + + + + The internal key used for the section + + + + + Human readable label for the section + + + + + Whether the section should be expanded by default or not. If not, the default set values are used + + + @@ -579,6 +618,20 @@ + + + =16.04``, the default is ``exit_code`` - legacy tools default to ``default`` described above. +]]> + + + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. @@ -1072,7 +1125,171 @@ - + + + tag set - it contains a set of tags. + ]]> + + + + + + + + + tag set ( used only in "data_source" tools ) - the external data source application may send back parameter names like "GENOME" which must be translated to "dbkey" in Galaxy.]]> + + + + + + + + Each of these maps directly to a remote_name value + + + + + + + The string representing the name of the parameter in the remote data source + + + + + + + The default value to use for galaxy_name if the remote_name parameter is not included in the request + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + tag set if galaxy_name="URL" - some remote data sources ( e.g., Gbrowse, Biomart ) send parameters back to Galaxy in the initial response that must be added to the value of "URL" prior to Galaxy sending the secondary request to the remote data source via URL.]]> + + + + + + + + + + + + + + + + + + + + + + + + tag set - allows for appending a param name / value pair to the value of URL. + +Example: + +```xml + + + + + + + +``` +]]> + + + + + + + + + + + + + + + + tag set the parameter value received from a remote data source may be named differently in Galaxy, and this tag set allows for the value to be appropriately translated.]]> + + + + + + + + + tag set - allows for changing the data type value to something supported by Galaxy. + +Example: + +```xml + + + + + + + +``` +]]> + + + + + + + + + + + + + Galaxy currently supports using regular expressions to scan stdout and stderr, and it also allows exit codes to be scanned for ranges. The 'stdio' tag has two subtags, 'regex' and 'exit_code', to define regular expressions and exit code processing, respectively. @@ -1142,7 +1359,7 @@ - + Documentation for ChangeFormat From 63c019b13b44fedc9159909eb7995294d3653244 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 18:22:12 -0400 Subject: [PATCH 063/109] More progress... - Fix to previous commit. - Remove downstream config file type that was abandoned work and not in galaxy-core. - More documentation for configfile stuff. - Improvements to representation for output and santizer. --- galaxy.xsd | 198 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 152 insertions(+), 46 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index c65a7ff986d6..f5d95f5bc7c0 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -44,6 +44,7 @@ The outer-most tag set of tool XML files. Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. + @@ -785,70 +786,144 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - Documentation for Sanitizer + tag set - it contains a set of and tags. + +Examples: + +``` + + + + + +``` + +``` + + + + + + + + +``` +]]> - + - + - Documentation for sanitize + is this parameter sanitized (default is ``true``) - + - Documentation for invalid_char + character to replace invalid characters with - + + + + + + + - Documentation for Valid + tag set. Used to specify a list of allowed characters. Contains and tags.]]> - + - Documentation for initial + initial characters to allow +no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - + - Documentation for ValidElementType + - Documentation for preset + Target characters from the list of valid characters (e.g. ``string.printable``). - Documentation for value + A character to add or remove from the list of valid characters - + - Documentation for Add + Used to add individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). - + - + - Documentation for Remove + Used to remove individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). - + - + - - + + + + + + + tag set. Used to specify a mapping of disallowed character to replacement string. Contains and tags.]]> + + + + + + + initial character mapping (default is ``galaxy.util.mapped_chars``) + + + + + + + + + + Replace all occurrences of this character with the string of ``target``. + + + + + Replace all occurrences of ``source`` with this string + + + + + + + + + + Character to remove from mapping. + + + + + + + @@ -908,16 +983,23 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - Documentation for Outputs + tag set. The files and collections created by tools as a result of their execution are named by Galaxy. You specify the number and type of your output files using the contained and tags. You must pass them to your tool executable through using line variables just like the parameters described in the previous sections. +]]> - - + + + + + + + - Documentation for Data + TODOWIKIMOVE @@ -1099,15 +1181,55 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - Documentation for ConfigFiles + tag sets - it defines an additional configuration section.]]> + - Documentation for ConfigFile + tag set. It allows for the creation of a temporary file for file-based parameter transfer. + +*Example* + +The following is taken from the ~/tools/plotting/xy_plot.xml tool config. + +```xml + + + ## Setup R error handling to go to stderr + options( show.error.messages=F, error = function () { cat( geterrmessage(), file=stderr() ); q( "no", 1, F ) } ) + ## Determine range of all series in the plot + xrange = c( NULL, NULL ) + yrange = c( NULL, NULL ) + #for $i, $s in enumerate( $series ) + s${i} = read.table( "${s.input.file_name}" ) + x${i} = s${i}[,${s.xcol}] + y${i} = s${i}[,${s.ycol}] + xrange = range( x${i}, xrange ) + yrange = range( y${i}, yrange ) + #end for + ## Open output PDF file + pdf( "${out_file1}" ) + ## Dummy plot for axis / labels + plot( NULL, type="n", xlim=xrange, ylim=yrange, main="${main}", xlab="${xlab}", ylab="${ylab}" ) + ## Plot each series + #for $i, $s in enumerate( $series ) + #if $s.series_type['type'] == "line" + lines( x${i}, y${i}, lty=${s.series_type.lty}, lwd=${s.series_type.lwd}, col=${s.series_type.col} ) + #elif $s.series_type.type == "points" + points( x${i}, y${i}, pch=${s.series_type.pch}, cex=${s.series_type.cex}, col=${s.series_type.col} ) + #end if + #end for + ## Close the PDF file + devname = dev.off() + + +``` +]]> @@ -1116,15 +1238,10 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for name - - - Documentation for type - - - + - - - - Documentation for ConfigFileType - - - - - - - Documentation for ChangeFormat From 1125f1b8ea8b993ac5493f663b76c4ba54350ee4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 18:59:58 -0400 Subject: [PATCH 064/109] Simplify schema and improve docs related to sanitization. --- galaxy.xsd | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index f5d95f5bc7c0..76b875c9f1fb 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -845,41 +845,40 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - + - + Used to add individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). - Target characters from the list of valid characters (e.g. ``string.printable``). + Add target characters from the list of valid characters (e.g. ``string.printable``). - A character to add or remove from the list of valid characters + Add a character to add to the list of valid characters - - - Used to add individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). - - - - - Used to remove individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). - - - + + + Remove characters from the list of valid characters (e.g. ``string.printable``). + + + + + A character to remove from the list of valid characters + + - - + + @@ -895,9 +894,9 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - + - + @@ -910,9 +909,9 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - + - + From 21cd752e252d17fdba2401cbe78bc3ef62ffd327 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 19:18:51 -0400 Subject: [PATCH 065/109] Improve docs on filter and change_format of outputs. --- galaxy.xsd | 80 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 76b875c9f1fb..f532dd8dd099 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1004,7 +1004,7 @@ Container tag set for the tag set. The files and collections created by t - Documentation for filter + @@ -1068,6 +1068,44 @@ Container tag set for the tag set. The files and collections created by t + + + + `` tag can contain a ```` tag which includes a Python code block to be executed to test whether to include this output in the outputs the tool ultimately creates. If the code, when executed, returns ``True``, the output dataset is retained. In these code blocks the tool parameters appear as Python variables and are thus referred to without the $ used for the Cheetah template (used in the ```` tag). Variables that are part of conditionals are accessed using a hash named after the conditional. For example: + +```xml + + + + + + + + + + + + + + + + + + + + + Documentation for DiscoverDatasets @@ -1467,30 +1505,56 @@ The target value (e.g. for setting data format: the list of supported data forma - Documentation for ChangeFormat + tag set and is the container tag set for the following tag set.]]> - + - + - Documentation for WhenData + + + + +``` + +Then whenever the user selects the "interval"" option from the select list, the following structure in your tool config will override the format="fasta" setting in the tag set with format="interval". + +```xml + + + + + + + +``` +]]> - Documentation for input + This value must be the attribute name of the desired input parameter - Documentation for value + This value must also be an attribute name of an input parameter - Documentation for format + This value must be a supported data type (e.g. ``format="interval"``) From 804c072d5af2e5704d57a503a84ad6733f03c1c0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 21:14:32 -0400 Subject: [PATCH 066/109] And more progress... - Docs for help. - Docs for tests. - Docs for code files. - Implementing more of the assert_xxx test flags. - Improved coverage of test assertion element descriptions. --- galaxy.xsd | 489 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 429 insertions(+), 60 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index f532dd8dd099..dd47c21d836f 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -44,8 +44,8 @@ The outer-most tag set of tool XML files. Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. - + @@ -53,7 +53,56 @@ The outer-most tag set of tool XML files. - Documentation for help + + +.. class:: warningmark + +'''TIP''' This tool requires *fasta* format. + +---- + +'''Example''' + +Query sequence:: + >seq1 + >ATCG... + +.. image:: my_figure.png + :height: 500 + :width: 600 + + +``` + +Best Practices + +From https://galaxy-iuc-standards.readthedocs.org/en/latest/ : + +The help tag should be started and finished by a CDATA tag. + +``` + +]]> @@ -150,11 +199,11 @@ The outer-most tag set of tool XML files. - Documentation for Code + **Deprecated** do not use this unless absolutely necessary. This tag set provides detailed control of the way the tool is executed. This (optional) code can be deployed in a separate file in the same directory as the tool's config file. These hooks are being replaced by new tool config features and methods in the ~/lib/galaxy/tools/__init__.py code file. - + - Documentation for file + This value is the name of the executable code file, and is called in the exec_before_process(), exec_before_job(), exec_after_process() and exec_after_job()( methods. @@ -170,7 +219,9 @@ The outer-most tag set of tool XML files. - Documentation for Tests + tag sets. Any number of tests can be included, and each test is wrapped within separate tag sets. Functional tests are executed via Planemo or the ~/run_tests.sh shell script. + +See [WritingTests](https://wiki.galaxyproject.org/Admin/Tools/WritingTests) on the wiki and [Test-Driven Development](http://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development) section of the Planemo docs for more information on writing tests.]]> @@ -178,39 +229,123 @@ The outer-most tag set of tool XML files. - Documentation for Test + + + + + + + + + + + + + + + + +Example + +Test the execution of the MAF-to-FASTA converter ( ~/tools/maf/maf_to_fasta.xml ). + + +Toggle line numbers + + + + + + + + +Example + +This test demonstrates verifying specific properties about a test output instead of directly comparing it to another file. Here the file attribute is not specified and instead a series of assertions is made about the output. + + +Toggle line numbers + + + + + + + + + + + + + + + +]]> + + + Describe assertions about the jobs standard error. + + + + + Describe assertions about the jobs standard error. + + + + + Describe assertions about the jobs standard error. + + + + + Describe the job's expected exit code. + + + + + Expect the job to fail, if set to true no job output checks may be present in test definition. + + - Documentation for TestParam + This tag set defines the tool's input parameters for executing the tool via the functional test framework. - Documentation for name + This value must match the name of the associated input parameter. - Documentation for value + This value must be one of the legal values that can be assigned to an input parameter. - Documentation for ftype + This attribute name should be included only with the parameter that defines the input dataset for the tool. If this attribute name is not included, the functional test framework will attempt to determine the data type for the input dataset using the data type sniffers. - Documentation for dbkey + dbkey for the referenced input format. @@ -282,26 +417,91 @@ The outer-most tag set of tool XML files. - - - - + + + + ``).]]> + + + + + + ``).]]> + + + + + + `` ).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + ``).]]> + + + + + + `` ).]]> + + + + + + ``).]]> + + + + + `` ||).]]> + + - - - - Line the test output must contain. - - - - - - - Text the test output must contain. - - - @@ -1002,11 +1202,8 @@ Container tag set for the tag set. The files and collections created by t - - - - - + + @@ -1050,7 +1247,7 @@ Container tag set for the tag set. The files and collections created by t Documentation for Collection - + @@ -1106,24 +1303,109 @@ The ```` tag can contain a ```` tag which includes a Python code b - + - Documentation for DiscoverDatasets + Describe datasets to dynamically collect after the job complete. + +More information can be found at https://wiki.galaxyproject.org/Admin/Tools/MultipleOutputFiles#Number_of_Output_datasets_cannot_be_determined_until_tool_run. + - Documentation for pattern + Regular expression used to find filenames and parse dynamic properties. - Documentation for directory + Directory (relative to working directory) to search for files. + + + + + Format (or datatype) of discovered datasets (an alias with ``ext``). + + + + + Format (or datatype) of discovered datasets (an alias with ``format``). + + + + + Indication if this dataset is visible in output history. This defaults to ``false``, but probably shouldn't - be sure to set to ``true`` if that is your intention. + + + + + Replace the primary dataset described by the parameter ``data`` parameter with the first output discovered. + + + + + + See https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_list_of_pairs.xml and https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_nested.xml for examples + + + + Regular expression used to find filenames and parse dynamic properties. + + + + + Directory (relative to working directory) to search for files. + + + + + Format (or datatype) of discovered datasets (an alias with ``ext``). + + + + + Format (or datatype) of discovered datasets (an alias with ``format``). + + + + + Indication if this dataset is visible in output history. This defaults to ``false``, but probably shouldn't - be sure to set to ``true`` if that is your intention. - Documentation for Actions + `` in the Bowtie wrapper is used in lieu of the deprecated tag to set the dbkey of the output dataset. In bowtie_wrapper.xml (see below), according to the first action block, if the refGenomeSource.genomeSource is "indexed" (not "history"), then it will assign the dbkey of the output file to be the same as that of the reference file. It does this by looking at through the loc file and finding the line that has the value that's been selected in the index dropdown box as column 1 of the loc file entry and using the dbkey, in column 0 (ignoring comment lines (starting with #) along the way). + +If refGenomeSource.genomeSource is "history", it resorts to default behavior for Galaxy, which is that the output is assigned the same value as the first input that has a dbkey specified. + +The second block would not be needed for most cases--it is required here to handle the specific case of a small reference file we use for functional testing. It says that if the dbkey has been set to "equCab2chrM" (that's what the tag) does then it should be changed to "equCab2" (the @@ -1137,63 +1419,63 @@ The ```` tag can contain a ```` tag which includes a Python code b - Documentation for Action + - Documentation for type + - Documentation for name + - Documentation for default + - Documentation for ActionsOption + - Documentation for type + - Documentation for name + - Documentation for column + - Documentation for offset + - Documentation for param_attribute + - Documentation for ActionsConditional + @@ -1205,14 +1487,14 @@ The ```` tag can contain a ```` tag which includes a Python code b - Documentation for ActionsConditionalWhen + - Documentation for value + @@ -1446,7 +1728,13 @@ The target value (e.g. for setting data format: the list of supported data forma - Galaxy currently supports using regular expressions to scan stdout and stderr, and it also allows exit codes to be scanned for ranges. The 'stdio' tag has two subtags, 'regex' and 'exit_code', to define regular expressions and exit code processing, respectively. + `` tag has two subtags, ```` and ````, to define regular expressions and exit code processing, respectively. They are defined below. If a tool does not have any valid ```` or ```` tags, then Galaxy will use the previous technique for finding errors. + +A note should be made on the order in which exit codes and regular expressions are applied and how the processing stops. Exit code rules are applied before regular expression rules. The rationale is that exit codes are more clearly defined and are easier to check computationally, so they are applied first. Exit code rules are applied in the order in which they appear in the tool's configuration file, and regular expressions are also applied in the order in which they appear in the tool's configuration file. However, once a rule is triggered that causes a fatal error, no further rules are checked.]]> @@ -1457,21 +1745,51 @@ The target value (e.g. for setting data format: the list of supported data forma - Documentation for ExitCode + tag defines a range of exit codes, and each range can be associated with a description of the error (e.g., "Out of Memory", "Invalid Sequence File") and an error level. The description just describes the condition and can be anything. The error level is either a warning or a fatal error. A warning means that stderr will be updated with the error's description. A fatal error means that the tool's execution will be marked as having an error and the workflow will stop. Note that, if the error level is not supplied, then a fatal error is assumed to have occurred. + +The exit code's range can be any consecutive group of integers. More advanced ranges, such as noncontiguous ranges, are currently not supported. Ranges can be specified in the form "m:n", where m is the start integer and n is the end integer. If ":n" is specified, then the exit code will be compared against all integers less than or equal to n. If "m:" is used, then the exit code will be compared against all integers greater than or equal to m. If the exit code matches, then the error level is applied and the error's description is added to stderr. If a tool's exit code does not match any of the supplied tags' ranges, then no errors are applied to the tool's execution. + +Note that most Unix and Linux variants only support positive integers 0 to 255 for exit codes. If an exit code falls out of the range 0 to 255, the usual convention is to only use the lower 8 bits for the exit code. The only known exception is if a job is broken into subtasks using the tasks runner and one of those tasks is stopped with a POSIX signal. (Note that signals should be used as a last resort for terminating processes.) In those cases, the task will receive -1 times the signal number. For example, suppose that a job uses the tasks runner and 8 tasks are created for the job. If one of the tasks hangs, then a sysadmin may choose to send the "kill" signal, SIGKILL, to the process. In that case, the task (and its job) will exit with an exit code of -9. More on POSIX signals can be found at http://en.wikipedia.org/wiki/Unix_signal as well as man pages on "signal". + +The tag's supported attributes are as follows: + +* range: This indicates the range of exit codes to check. The range can be one of the following: + * n: the exit code will only be compared to n; + * [m:n]: the exit code must be greater than or equal to m and less than or equal to n; + * [m:]: the exit code must be greater than or equal to m; + * [:n]: the exit code must be less than or equal to n. +* level: This indicates the error level of the exit code. The level can have one of two values: + * warning: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A warning-level error will not cause the tool to fail. + * fatal: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. +* description: This is an optional description of the error that corresponds to the exit code. +The following is an example of the tag: + +```xml + + + + + +``` + +If the tool returns 0 or 1, then the tool will not be marked as having an error. If the exit code is 2, then the tool will fail with the description "Out of Memory" added to stderr. If the tool returns 3, 4, or 5, then the tool will not be marked as having failed, but "Low disk space" will be added to stderr. Finally, if the tool returns any number greater than or equal to 6, then the description "Bad input dataset" will be added to stderr and the tool will be marked as having failed. + +]]> - Documentation for range + - Documentation for level + - Documentation for description + @@ -1479,7 +1797,58 @@ The target value (e.g. for setting data format: the list of supported data forma - A regular expression defines a pattern of characters + tag does not contain the match attribute, then the tag will be ignored. The regular expression can be any valid Python regular expression. All regular expressions are performed case insensitively. For example, if match contains the regular expression "actg", then the regular expression will match against "actg", "ACTG", "AcTg", and so on. Also note that, if double quotes (") are to be used in the match attribute, then the value " can be used in place of double quotes. Likewise, if single quotes (') are to be used in the match attribute, then the value ' can be used if necessary. +* level: This works very similarly to the tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. For example, if stdout matches on a regular expression, then the regular expression's description is added to the beginning of stdout (instead of stderr). The level can be log, warning or fatal as described below. + * log and warning: If the regular expression matches against its source input (i.e., stdout and/or stderr), then a description of the error will be added to the beginning of the source, prepended with either 'Log:' or 'Warning:'. A log-level/warning-level error will not cause the tool to fail. + * fatal: If the regular expression matches against its source input, then a description of the error will be added to the beginning of the source. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. +* description: Just like its counterpart, this is an optional description of the regular expression that has matched. + +The following is an example of regular expressions that may be used: + +```xml + + + + + + +``` + +The regular expression matching proceeds as follows. First, if either stdout or stderr match on "low space", then a warning is registered. If stdout contained the string "---LOW SPACE---", then stdout has the string "Warning: Low space on device" added to its beginning. The same goes for if stderr had contained the string "low space". Since only a warning could have occurred, the processing continues. + +Next, the regular expression "error" is matched only against stdout. If stdout contains the string "error" regardless of its capitalization, then a fatal error has occurred and the processing stops. In that case, stdout would be prepended with the string "Fatal: Unknown error encountered". Note that, if stderr contained "error", "ERROR", or "ErRor" then it would not matter - stderr was not being scanned. + +If the second regular expression did not match, then the third regular expression is checked. The third regular expression does not contain an error level, so an error level of "fatal" is assumed. The third regular expression also does not contain a source, so both stdout and stderr are checked. The third regular expression looks for 12 consecutive "C"s or "G"s in any order and in uppercase or lowercase. If stdout contained "cgccGGCCcGGcG" or stderr contained "CCCCCCgggGGG", then the regular expression would match, the tool would be marked with a fatal error, and the stream that contained the 12-nucleotide CG island would be prepended with "Fatal: Fatal error - CG island 12 nts long found". + +Finally, if the tool did not match any of the fatal errors, then the fourth regular expression is checked. Since no source is specified, both stdout and stderr are checked. If "Branch A" is at the beginning of stdout or stderr, then a warning will be registered and the source that contained "Branch A" will be prepended with the warning "Warning: Branch A was taken in execution". +]]> From 23449750d46e4df208fc46d1c2a4d3840cd080f4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 22:46:56 -0400 Subject: [PATCH 067/109] More progress... - Refine test outputs and include syntax for testing extra files and md5sums. --- galaxy.xsd | 69 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index dd47c21d836f..48420ace3c1c 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -327,6 +327,7 @@ Toggle line numbers + @@ -349,6 +350,21 @@ Toggle line numbers + + + Define extra composite input files for test input. + + + + Path relative to test-data of composite file. + + + + + Optional datatype of composite file for test input. + + + Definition of a collection for test input. @@ -368,6 +384,7 @@ Toggle line numbers + @@ -389,13 +406,37 @@ Toggle line numbers Documentation for sort + + + An alias for file. + + + + + Expected MD5sum for output. + + + + + + Define test for extra files on corresponding output. + + + + + + Extra file type (either ``file`` or ``directory``). + + + + - Documentation for TestOutputCollection + Define test for extra files on corresponding output collection. - + @@ -502,21 +543,6 @@ Toggle line numbers - - - - - - - Documentation for file - - - - - Documentation for name - - - Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. @@ -1211,7 +1237,7 @@ Container tag set for the tag set. The files and collections created by t Documentation for name - + Documentation for format @@ -1256,7 +1282,7 @@ Container tag set for the tag set. The files and collections created by t - Documentation for format + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). @@ -1264,6 +1290,11 @@ Container tag set for the tag set. The files and collections created by t Documentation for label + + + Input collection or dataset to derive output dataset element's format/datatype from. + + From 3d3c3fee4f576408c223c723b57b003ae6ccf4c1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 23:47:08 -0400 Subject: [PATCH 068/109] More progress. - Add strict to command. - Add conditional and repeat to test params. - Add count to output collections. - Implement more collection output attributes (type_source, etc...). - Reduce need for type on collection defs (since type_source can be defined). --- galaxy.xsd | 51 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 48420ace3c1c..24d7dd062bc5 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -291,7 +291,7 @@ Toggle line numbers ]]> - + @@ -321,6 +321,25 @@ Toggle line numbers + + + + + + + + + + + + + + + + + + + This tag set defines the tool's input parameters for executing the tool via the functional test framework. @@ -448,6 +467,11 @@ Toggle line numbers Documentation for type + + + Number of elements in output collection. + + @@ -864,6 +888,11 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. + + + + + @@ -1165,7 +1194,7 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"``Documentation for name - + Documentation for column @@ -1280,7 +1309,7 @@ Container tag set for the tag set. The files and collections created by t Documentation for name - + Collection type for output (e.g. ``paired``, ``list``, or ``list:list``). @@ -1295,6 +1324,22 @@ Container tag set for the tag set. The files and collections created by t Input collection or dataset to derive output dataset element's format/datatype from. + + + Input collection to derive collection's type (e.g. ``collection_type``) from. + + + + + + Input collection or dataset to derive structure of input from (out element count and identifiers). + + + + + If ``structured_like`` is set, inherit format of outputs from format of corresponding input. + + From 80d086b861f1401568b967d96e2e37461ca677a2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 7 Sep 2016 23:54:15 -0400 Subject: [PATCH 069/109] More progress... - Allow section in test parameters. - Implement basic metadata testing. --- galaxy.xsd | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 24d7dd062bc5..735d1ff9ace4 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -326,8 +326,16 @@ Toggle line numbers + + + + + + + + @@ -404,6 +412,7 @@ Toggle line numbers + @@ -436,6 +445,18 @@ Toggle line numbers + + + + Documentation for name + + + + + Documentation for value + + + Define test for extra files on corresponding output. From 19b273cdf55df6b2593715d7daaf4690d8d6935b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Sep 2016 08:27:56 -0400 Subject: [PATCH 070/109] More progress... - Allow nested elements in test output collections (hackily) - Implement ``compare`` and ``delta`` for test output comparison. - Implement the new ``checksum`` output test. --- galaxy.xsd | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 735d1ff9ace4..22b08e8ca6d6 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -410,6 +410,9 @@ Toggle line numbers Documentation for TestOutput + + @@ -444,6 +447,18 @@ Toggle line numbers Expected MD5sum for output. + + + Checksum expression for output in the form ``hash_type:hash_value`` (e.g. ``sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041``). + + + + + + + If ``compare`` is set to ``diff``, the number of lines of difference to allow (each line with a modification is a line added and a line removed so this counts as two lines). If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed. + + @@ -2194,6 +2209,18 @@ Then whenever the user selects the "interval"" option from the select list, the + + + Type of comparison to use for test outputs. + + + + + + + + + Documentation for PermissiveBoolean From 7f5cc04c114bd5e696e7e54697e5a7bf58277fa7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Sep 2016 11:35:33 -0400 Subject: [PATCH 071/109] More progress... - Add tool action. - Stub workflow_compatible and profile_version attributes on tool. - Add many more attributes to parallelism tag. - Add maxseconds and expect_num_outputs to tool test tag. - Small fix to block in test cases. - Implement assigning metadata in test params. - Allow more versitle ordering of test blocks (too permissive now). - Implement testing of output metadata and discovered datasets. - Add nginx_upload attribute used by upload.xml and various conditional attributes for pulling in values from Galaxy internals. - Implement the stub of a ``upload_dataset`` param and say not to use it. - Add color and library data parameter types. - More attributes to filters and validators. - Implement auto_format on outputs. - Implement filters and explicit data outputs on output collections. - Implement inputs as json element. - Add empty dataset validator. --- galaxy.xsd | 221 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 207 insertions(+), 14 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 22b08e8ca6d6..933709e484bc 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -44,6 +44,7 @@ The outer-most tag set of tool XML files. Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. + @@ -134,6 +135,16 @@ The help tag should be started and finished by a CDATA tag. Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. + + + Documentation for profile_version + + + + + Is this tool usable within a workflow (defaults to ``true`` for normal tools and ``false`` for data sources). + + Only if "tool_type" attribute value is "data_source" - defines the HTTP request method to use when communicating with an external data source application ( the default is "get" ). @@ -141,7 +152,19 @@ The help tag should be started and finished by a CDATA tag. - + + + + Describe the backend Python action to execute for this Galaxy tool. + + + + + + + + + This is a container tag set for the 'requirement' tag set described below. @@ -196,6 +219,26 @@ The help tag should be started and finished by a CDATA tag. Documentation for method + + + Documentation for merge_outputs + + + + + Documentation for split_inputs + + + + + Documentation for split_size + + + + + Documentation for split_mode + + @@ -315,11 +358,21 @@ Toggle line numbers Describe the job's expected exit code. + + + Assert the number of outputs this test should produce. + + Expect the job to fail, if set to true no job output checks may be present in test definition. + + + Maximum amount of time to let test run. + + @@ -347,6 +400,7 @@ Toggle line numbers + @@ -355,6 +409,7 @@ Toggle line numbers + @@ -410,14 +465,10 @@ Toggle line numbers Documentation for TestOutput - - - - - + - + + Documentation for name @@ -460,7 +511,19 @@ Toggle line numbers - + + + + + + + + + + + + + Documentation for name @@ -472,6 +535,32 @@ Toggle line numbers + + + + Documentation for name + + + + + Documentation for value + + + + + + Define test for a discovered dataset on corresponding output. + + + + + + The designation of the discovered dataset. + + + + + Define test for extra files on corresponding output. @@ -630,6 +719,11 @@ Toggle line numbers Documentation for target + + + Boolean indicates if this is an upload tool or not. + + @@ -639,6 +733,11 @@ Toggle line numbers + + + Internal, intentionally undocumented feature. + + Documentation for display @@ -657,8 +756,8 @@ Toggle line numbers - - + + This is a container for conditional parameters in the tool ( must contain 'when' tag sets ) - the command line is wrapped in an if-else statement. @@ -666,9 +765,30 @@ Toggle line numbers - + + + + + + + + + + + + + + + + @@ -885,8 +1005,10 @@ Toggle line numbers + + @@ -1039,6 +1161,11 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for message + + + Documentation for check + + Documentation for filename @@ -1074,6 +1201,11 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for split + + + Documentation for skip + + @@ -1270,6 +1402,11 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"``Documentation for ref_attribute + + + Documentation for index + + @@ -1293,7 +1430,7 @@ Container tag set for the tag set. The files and collections created by t - + @@ -1302,6 +1439,11 @@ Container tag set for the tag set. The files and collections created by t Documentation for name + + + If ``true``, this output will sniffed and its format determined automatically by Galaxy. + + Documentation for format @@ -1338,7 +1480,9 @@ Container tag set for the tag set. The files and collections created by t Documentation for Collection + + @@ -1615,10 +1759,17 @@ Example: tag sets - it defines an additional configuration section.]]> - + + + + + + + + + + + tag set. It tells Galaxy to write out a JSON representation of the tool parameters . + +*Example* + +The following will create a cheetah variable that can be evaluated as ``$inputs`` that +will contain the tool parameter inputs. + +```xml + + + +``` + +The following will instead write the inputs to the tool's working directory with the specified name (i.e. ``inputs.json``). + +```xml + + + +``` +]]> + + + + + + Cheetah variable to populate the path to the inputs JSON file created in response to this directive. + + + + + Path relative to the working directory of the tool for the inputs JSON file created in response to this directive. + + + + + + Documentation for ValidatorType + From 49ff357854ff1a8dab9d792c0d97881f33c5a603 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Sep 2016 12:54:11 -0400 Subject: [PATCH 072/109] Stub out environment variables and add a new tool type. --- galaxy.xsd | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/galaxy.xsd b/galaxy.xsd index 933709e484bc..eba1f053e237 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -45,6 +45,7 @@ The outer-most tag set of tool XML files. + @@ -1754,6 +1755,34 @@ Example: + + + + + + + + + + + + + + + + + + + + Documentation for name + + + + + + + tag sets - it defines an additional configuration section.]]> @@ -2255,6 +2284,7 @@ Then whenever the user selects the "interval"" option from the select list, the + From c6ae6ee8cb3c240923e4738f0e8f0a4a1134cbd5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Sep 2016 12:59:23 -0400 Subject: [PATCH 073/109] Embed citations right in galaxy.xsd. Not that I mind splitting it up - but it will make a couple things easier downstream and those few tags weren't enough to warrent breaking things up like that. --- citation.xsd | 30 ------------------------------ citations.xsd | 15 --------------- galaxy.xsd | 39 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 48 deletions(-) delete mode 100644 citation.xsd delete mode 100644 citations.xsd diff --git a/citation.xsd b/citation.xsd deleted file mode 100644 index 81c470687fb5..000000000000 --- a/citation.xsd +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. - - - - - - type of citation - - - - - - - - - Type of citation represented. - - - - - - - - diff --git a/citations.xsd b/citations.xsd deleted file mode 100644 index 3db22e7c9506..000000000000 --- a/citations.xsd +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. - - - - - - - - diff --git a/galaxy.xsd b/galaxy.xsd index eba1f053e237..7b63fc24e865 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -9,9 +9,6 @@ A Galaxy XML tool wrapper - - - + + + + Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + + + + + Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + + + + + + type of citation + + + + + + + + + Type of citation represented. + + + + + + + Documentation for RequirementType From dcfdd3ceaa62ccee0a8d22c6d299eab2742df47f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 8 Sep 2016 15:02:15 -0400 Subject: [PATCH 074/109] Many more fixes derived from testing tools-iuc. --- galaxy.xsd | 248 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 163 insertions(+), 85 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 7b63fc24e865..14ba121340a3 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -21,7 +21,7 @@ The outer-most tag set of tool XML files. name="UCSC Main" version="1.0.0" hidden="false" - profile_version="16.01" + profile="16.01" tool_type="data_source" URL_method="post"> ``` @@ -29,6 +29,8 @@ The outer-most tag set of tool XML files. + + @@ -133,7 +135,7 @@ The help tag should be started and finished by a CDATA tag. Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. - + Documentation for profile_version @@ -378,6 +380,23 @@ Toggle line numbers + + + + + Describe assertions about the jobs standard error. + + + + + Describe assertions about the jobs standard error. + + + + + Describe assertions about the jobs standard error. + + @@ -503,9 +522,14 @@ Toggle line numbers + + + If ``compare`` is set to ``diff``, the number of lines of difference to allow (each line with a modification is a line added and a line removed so this counts as two lines). + + - If ``compare`` is set to ``diff``, the number of lines of difference to allow (each line with a modification is a line added and a line removed so this counts as two lines). If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed. + If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed. @@ -578,7 +602,7 @@ Toggle line numbers Define test for extra files on corresponding output collection. - + @@ -748,7 +772,7 @@ Toggle line numbers Documentation for InputType - + Name for this element @@ -787,6 +811,13 @@ Referenced parameter to pass method.]]> Is referenced parameter is the same group.]]> + + + + Human readable description for the conditional, unused in the Galaxy UI currently. + + + @@ -861,7 +892,7 @@ Is referenced parameter is the same group.]]> Human readable label for the section - + Whether the section should be expanded by default or not. If not, the default set values are used @@ -875,23 +906,24 @@ Is referenced parameter is the same group.]]> - - - Documentation for label - - - - - - - - + + Documentation for type + + + Documentation for type + + + + + Command-line argument this parameter maps to (e.g. ``--foobar``). + + Documentation for label @@ -912,12 +944,12 @@ Is referenced parameter is the same group.]]> Documentation for optional - + Documentation for min - + Documentation for max @@ -982,7 +1014,8 @@ Is referenced parameter is the same group.]]> Documentation for falsevalue - + + Documentation for size @@ -995,6 +1028,26 @@ Is referenced parameter is the same group.]]> + + + + + + Documentation for label + + + + + + + + + Documentation for help + + + + + Documentation for ParamType @@ -1149,61 +1202,66 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for Validator - - - Documentation for type - - - - - Documentation for message - - - - - Documentation for check - - - - - Documentation for filename - - - - - Documentation for metadata_name - - - - - Documentation for metadata_column - - - - - Documentation for line_startswith - - - - - Documentation for min - - - - - Documentation for max - - - - - Documentation for split - - - - - Documentation for skip - - + + + + + Documentation for type + + + + + Documentation for message + + + + + Documentation for check + + + + + Documentation for filename + + + + + Documentation for metadata_name + + + + + Documentation for metadata_column + + + + + Documentation for line_startswith + + + + + Documentation for min + + + + + Documentation for max + + + + + Documentation for split + + + + + Documentation for skip + + + + + @@ -1422,16 +1480,22 @@ Container tag set for the tag set. The files and collections created by t + + + + + + + + TODOWIKIMOVE - - - - + + Documentation for name @@ -1473,14 +1537,21 @@ Container tag set for the tag set. The files and collections created by t + + + + + + + + + Documentation for Collection - - - + @@ -2067,11 +2138,16 @@ With ``stdio`` directive, Galaxy can use regular expressions to scan stdout and A note should be made on the order in which exit codes and regular expressions are applied and how the processing stops. Exit code rules are applied before regular expression rules. The rationale is that exit codes are more clearly defined and are easier to check computationally, so they are applied first. Exit code rules are applied in the order in which they appear in the tool's configuration file, and regular expressions are also applied in the order in which they appear in the tool's configuration file. However, once a rule is triggered that causes a fatal error, no further rules are checked.]]> - - + + + + + + + @@ -2301,6 +2377,7 @@ Then whenever the user selects the "interval"" option from the select list, the + @@ -2372,6 +2449,7 @@ Then whenever the user selects the "interval"" option from the select list, the + @@ -2452,7 +2530,7 @@ Then whenever the user selects the "interval"" option from the select list, the Documentation for RangeType - + From 8c2a4e12758827b0855181de962ef0f0a61515a7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Sep 2016 10:11:19 -0400 Subject: [PATCH 075/109] And more progress - now covers all Galaxy test tools and all IUC. - Add block. - Add block. - Add a newerish validator and allow filter/validator columns to be column names. - More permissives boolean values to match Galaxy in various places. - Other mostly obsecure features. --- galaxy.xsd | 108 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 14ba121340a3..e65150f0fc9d 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -108,6 +108,8 @@ The help tag should be started and finished by a CDATA tag. + + @@ -125,11 +127,16 @@ The help tag should be started and finished by a CDATA tag. This string defaults to "1.0.0' if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. - + Allows for tools to be loaded upon server startup, but not displayed in the tool menu + + + + + Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. @@ -252,7 +259,7 @@ The help tag should be started and finished by a CDATA tag. - Documentation for UIhints + Used only for data source tools, this directive contains UI options (currently only ``minwidth`` is valid). @@ -260,6 +267,48 @@ The help tag should be started and finished by a CDATA tag. + + + + This directive is used to specify some rarely modified options. + + + + Deprecated, likely unused attribute. + + + + + This attribute can be used to turn off all input sanitization for a tool. + + + + + + + This directive is used to specify some rarely modified trackster options. + + + + + + + + + + + + + + + + + + + + + + tag sets. Any number of tests can be included, and each test is wrapped within separate tag sets. Functional tests are executed via Planemo or the ~/run_tests.sh shell script. @@ -726,7 +775,7 @@ Toggle line numbers Documentation for action - + Documentation for check_values @@ -869,6 +918,11 @@ Is referenced parameter is the same group.]]> The default number of repeat units + + + Short help description for repeat element. + + @@ -974,11 +1028,16 @@ Is referenced parameter is the same group.]]> Documentation for accept_default - + Documentation for force_select + + + Documentation for use_header_names + + Documentation for display @@ -999,7 +1058,7 @@ Is referenced parameter is the same group.]]> Documentation for hierarchy - + Documentation for checked @@ -1025,6 +1084,11 @@ Is referenced parameter is the same group.]]> Documentation for metadata_name + + + Deprecated method to allow access to Python code to generate options for a select list. + + @@ -1037,7 +1101,7 @@ Is referenced parameter is the same group.]]> - + @@ -1128,7 +1192,7 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - + Documentation for Options @@ -1229,9 +1293,9 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for metadata_name - + - Documentation for metadata_column + This can be an integer index to the column or a column name. @@ -1259,6 +1323,11 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - Documentation for skip + + + Documentation for table_name + + @@ -1418,9 +1487,9 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"``Documentation for name - + - Documentation for column + This can be a column index or a column name. @@ -2442,20 +2511,6 @@ Then whenever the user selects the "interval"" option from the select list, the - - - Documentation for CheckedType - - - - - - - - - - - Documentation for ValidatorType @@ -2471,6 +2526,7 @@ Then whenever the user selects the "interval"" option from the select list, the + @@ -2566,6 +2622,8 @@ Then whenever the user selects the "interval"" option from the select list, the + + From ff6cca8635e6ce1c906dd316dcc0d6ab0bed7fb3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Sep 2016 13:01:29 -0400 Subject: [PATCH 076/109] Doc updates --- galaxy.xsd | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index e65150f0fc9d..da749f3b41c4 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -14,7 +14,7 @@ ]]> @@ -134,17 +130,17 @@ The help tag should be started and finished by a CDATA tag. - + Display the display the tool's graphical tool form by setting this to ``true``. - Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. + Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. - Documentation for profile_version + Documentation for profile From 491efe8c4951eedecc1635dbc972fb49ec883230 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Sep 2016 15:07:23 -0400 Subject: [PATCH 077/109] Doc improvements. --- galaxy.xsd | 125 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 44 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index da749f3b41c4..385895064802 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1,5 +1,5 @@ -Galaxy Schema A Galaxy XML tool wrapper - + +``` + +A ``data_source`` tool contains a few more relevant attributes. ```xml -``` +``` Best Practices @@ -110,47 +123,71 @@ The help tag should be started and finished by a [CDATA tag](https://en.wikipedi - Must be unique across all tools; should be lowercase and contain only letters, numbers, and underscores. It allows for tool versioning and metrics of the number of times a tool is used, among other things. + Must be unique across all tools; +should be lowercase and contain only letters, numbers, and underscores. +It allows for tool versioning and metrics of the number of times a tool is used, +among other things. - This string is what is displayed as a hyperlink in the tool menu + This string is what is displayed as a +hyperlink in the tool menu. - This string defaults to "1.0.0' if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. + This string defaults to ``1.0.0`` if it is not +included in the tag. It allows for tool versioning and should be changed with each new version +of the tool. - Allows for tools to be loaded upon server startup, but not displayed in the tool menu + Allows for tools to be loaded upon +server startup, but not displayed in the tool menu. This attribute should be +applied in the toolbox configuration instead and so should be considered +deprecated. + - Display the display the tool's graphical tool form by setting this to ``true``. + Disable the display the tool's +graphical tool form by setting this to ``false``. - Allows for certain framework functionality to be performed on certain types of tools. This is currently only used in "data_source" tools, but will undoubtedly be used with other tools in the future. + Allows for certain framework +functionality to be performed on certain types of tools. Normal tools that execute +typical command-line jobs do not need to specify this, special kinds of tools such +as [Data Source](https://wiki.galaxyproject.org/Admin/Internals/DataSources) and +[Data Manager](https://wiki.galaxyproject.org/Admin/Tools/DataManagers) tools should +set this to have values such as ``data_source`` or ``manage_data``. - Documentation for profile + This string specified the minimum Galaxy +version that should be required to run this tool. Certain legacy behaviors such +as using standard error content to detect errors instead of exit code are disabled +automatically if profile is set to any version newer than ``16.01``, such as +``16.04``. - Is this tool usable within a workflow (defaults to ``true`` for normal tools and ``false`` for data sources). + This attribute indicates if +this tool usable within a workflow (defaults to ``true`` for normal tools and +``false`` for data sources). - Only if "tool_type" attribute value is "data_source" - defines the HTTP request method to use when communicating with an external data source application ( the default is "get" ). + Only used if ``tool_type`` attribute value +is ``data_source`` - this attribute defines the HTTP request method to use when +communicating with an external data source application (the default is ``get``). @@ -177,7 +214,7 @@ The help tag should be started and finished by a [CDATA tag](https://en.wikipedi - + This tag set is contained within the 'requirements' tag set. Third party programs or modules that the tool depends upon (and which are not distributed with Galaxy) are included in this tag set. The intention is that when Galaxy starts it can check whether the required programs or modules are available, and if not this tool will not be loaded. @@ -401,22 +438,22 @@ Toggle line numbers Describe the job's expected exit code. - + Assert the number of outputs this test should produce. - + Expect the job to fail, if set to true no job output checks may be present in test definition. - + Maximum amount of time to let test run. - + @@ -674,7 +711,7 @@ Toggle line numbers - + ``).]]> @@ -792,8 +829,8 @@ Toggle line numbers - - + + @@ -866,9 +903,9 @@ Is referenced parameter is the same group.]]> - - - + + + Documentation for ConditionalWhen @@ -882,9 +919,9 @@ Is referenced parameter is the same group.]]> - - - + + + This is a container for any tag sets that can be contained within the 'inputs' tag set. When this is used, the tool will allow the user to add any number of additional sets of the contained parameters ( an "Add new 'title'" button will be displayed on the tool form ). @@ -947,8 +984,8 @@ Is referenced parameter is the same group.]]> Whether the section should be expanded by default or not. If not, the default set values are used - - + + Documentation for Param @@ -1130,8 +1167,8 @@ Is referenced parameter is the same group.]]> - - + + This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). @@ -1165,8 +1202,8 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - - + + Documentation for ParamOption @@ -1186,8 +1223,8 @@ For newer tools with ``profile_version>=16.04``, the default is ``exit_code`` - - - + + Documentation for Options @@ -1550,7 +1587,7 @@ Container tag set for the tag set. The files and collections created by t - + @@ -1993,7 +2030,7 @@ This tag set is contained within the tag set. It tells Galaxy to w *Example* -The following will create a cheetah variable that can be evaluated as ``$inputs`` that +The following will create a cheetah variable that can be evaluated as ``$inputs`` that will contain the tool parameter inputs. ```xml @@ -2206,14 +2243,14 @@ A note should be made on the order in which exit codes and regular expressions a - + - + - - + + - + tag set and is the container tag set for the following tag set.]]> From 94ad78da97b98eee05a5ff0f890eec0e518845b7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Sep 2016 16:16:23 -0400 Subject: [PATCH 078/109] Much richer documentation for a few tags including ``command``. --- galaxy.xsd | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 200 insertions(+), 3 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 385895064802..e205a7ae3ede 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -47,13 +47,35 @@ A ``data_source`` tool contains a few more relevant attributes. - The value is displayed in the tool menu immediately following the hyperlink for the tool + `` tag set described above). + +### Example + +```xml +table browser +``` +]]> - Specifies the command to be run in order to get the tool's version string. The resulting value will be found in the "Info" field of the history dataset. + tophat -version +``` +]]> @@ -1171,7 +1193,182 @@ Is referenced parameter is the same group.]]> - This tag specifies how Galaxy should invoke the tool's executable, passing its required input parameter values (the command line specification links the parameters supplied in the form with the actual tool executable). Any word inside it starting with a dollar sign ($) will be treated as a variable whose values can be acquired from one of three sources: parameters, metadata, or output files. After the substitution of variables with their values, the content is interpreted with Cheetah and finally given to the interpreter specified in the corresponding attribute (if any). + bed12ToBed6 -i '$input' > '$output' +``` + +A few things to note about even this simple example: + +* Input and output variables (boringly named ``input`` and ``output``) + are expanded into paths using the ``$`` Cheetah directive. +* Paths should be quoted so that the Galaxy database files may contain spaces. +* We are building up a shell script - so special characters like ``>`` can be used + (in this case the standard output of the bedtools call is written to the path + specified by ``'$output'``). + +The bed12ToBed6 tool can be found [here](https://github.com/galaxyproject/tools-iuc/blob/master/tools/bedtools/bed12ToBed6.xml). + +A more sophisticated bedtools example demonstrates the use of loops, conditionals, +and uses whitespace to make a complex command very readable can be found in +[annotateBed](https://github.com/galaxyproject/tools-iuc/blob/master/tools/bedtools/annotateBed.xml) +tool. + +```xml + +bedtools annotate + -i "${inputA}" + #if $names.names_select == 'yes': + -files + #for $bed in $names.beds: + "${bed.input}" + #end for + -names + #for $bed in $names.beds: + "${bed.inputName}" + #end for + #else: + #set files = '" "'.join( [ str( $file ) for $file in $names.beds ] ) + -files "${files}" + #set names = '" "'.join( [ str( $name.display_name ) for $name in $names.beds ] ) + -names "${names}" + #end if + $strand + $counts + $both + > "${output}" + +``` + +The following example (taken from [xpath](https://github.com/galaxyproject/tools-iuc/blob/master/tools/xpath/xpath.xml) tool) +uses an interpreted executable. In this case a Perl script is shipped with the +tool and the directory of the tool itself is referenced with ``$__tool_directory__``. + +```xml + + perl $__tool_directory__/xpath -q -e '$expression' '$input' > '$output' + +``` + +The following example demonstrates accessing metadata from datasets. Metadata values +(e.g., ``${input.metadata.chromCol}``) are acquired from the ``Metadata`` model associated +with the objects selected as the values of each of the relative form field +parameters in the tool form. Accessing this information is generally enabled using +the following feature components. + +A set of "metadata information" is defined for each supported data type (see the +``MetadataElement`` objects in the various data types classes in +[``/lib/galaxy/datatypes``](https://github.com/galaxyproject/galaxy/tree/dev/lib/galaxy/datatypes). +The ``DatasetFilenameWrapper`` class in the +[/lib/galaxy/tools/wrappers.py](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/tools/wrappers.py) +code file wraps a metadata collection to return metadata parameters wrapped +according to the Metadata spec. + +```xml + + #set genome = $input.metadata.dbkey + #set datatype = $input.datatype + mkdir -p output_dir && + python $__tool_directory__/extract_genomic_dna.py + --input "$input" + --genome "$genome" + #if $input.is_of_type("gff"): + --input_format "gff" + --columns "1,4,5,7" + --interpret_features $interpret_features + #else: + --input_format "interval" + --columns "${input.metadata.chromCol},${input.metadata.startCol},${input.metadata.endCol},${input.metadata.strandCol},${input.metadata.nameCol}" + #end if + --reference_genome_source $reference_genome_cond.reference_genome_source + #if str($reference_genome_cond.reference_genome_source) == "cached" + --reference_genome $reference_genome_cond.reference_genome.fields.path + #else: + --reference_genome $reference_genome_cond.reference_genome + #end if + --output_format $output_format_cond.output_format + #if str($output_format_cond.output_format) == "fasta": + --fasta_header_type $output_format_cond.fasta_header_type_cond.fasta_header_type + #if str($output_format_cond.fasta_header_type_cond.fasta_header_type) == "char_delimited": + --fasta_header_delimiter $output_format_cond.fasta_header_type_cond.fasta_header_delimiter + #end if + #end if + --output $output + +``` + +In additon to demonstrating accessing metadata, this example demonstrates: + +* ``$input.is_of_type("gff")`` which can be used to check if an input is of a + given datatype. +* ``#set datatype = $input.datatype`` which is the syntax for defining variables + in Cheetah. + + + +### Reserved Variables + +Galaxy provides a few pre-defined variables which can be used in your command line, +even though they don't appear in your tool's parameters. + +Name | Description +---- | ----------- +``$__tool_directory__`` | The directory the tool currently resides in (new in 15.03) +``$__new_file_path__`` | ``config/galaxy.ini``'s ``new_file_path`` value +``$__tool_data_path__`` | ``config/galaxy.ini``'s tool_data_path value +``$__root_dir__`` | Top-level Galaxy source directory made absolute via ``os.path.abspath()`` +``$__datatypes_config__`` | ``config/galaxy.ini``'s datatypes_config value +``$__user_id__`` | Email's numeric ID (id column of ``galaxy_user`` table in the database) +``$__user_email__`` | User's email address +``$__app__`` | The ``galaxy.app.UniverseApplication`` instance, gives access to all other configuration file variables (e.g. $__app__.config.output_size_limit). Should be used as a last resort, may go away in future releases. + +Additional runtime properties are available as environment variables. Since these +are not Cheetah variables (the values aren't available until runtme) these should likely +be escaped with a backslash (``\``) when appearing in ``command`` or ``configfile`` elements. + +Name | Description +---- | ----------- +``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime). + +See the [Planemo docs](http://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) +on the topic of ``GALAXY_SLOTS`` for more information and examples. + +### Attributes + +#### ``detect_errors`` + +If present on the ``command`` tag, this attribute can be one of: + +* ``default`` no-op fallback to ``stdio`` tags and erroring on standard error output (for legacy tools). +* ``exit_code`` error if tool exit code is not 0. (The @jmchilton recommendation). +* ``aggressive`` error if tool exit code is not 0 or either ``Exception:`` or ``Error:`` + appears in standard error/output. (The @bgruening recommendation). + +For newer tools with ``profile>=16.04``, the default behavior is ``exit_code``. +Legacy tools default to ``default`` behavior described above (erroring if the tool +produces any standard error output). + +See [PR 117](https://github.com/galaxyproject/galaxy/pull/117) for more implementation +information and discussion on the ``detect_errors`` attribute. + +#### ``interpreter`` + +Older tools may define an ``intepreter`` attribute on the command, but this is +deprecated and using the ``$__tool_directory__`` variable is superior. +]]> From e9f08b411a0bf4ef8620bc66fb7c10d61112fb49 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Sep 2016 16:54:43 -0400 Subject: [PATCH 079/109] More work on documentation - revise last commit and add repeats and sections. --- galaxy.xsd | 135 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 110 insertions(+), 25 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index e205a7ae3ede..3bc0e29ef88c 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -128,12 +128,12 @@ Query sequence:: ``` -Best Practices +### Best Practices -From https://galaxy-iuc-standards.readthedocs.org/en/latest/ : +From https://galaxy-iuc-standards.readthedocs.org/en/latest/: The help tag should be started and finished by a [CDATA tag](https://en.wikipedia.org/wiki/CDATA). - + ]]> @@ -820,34 +820,38 @@ Toggle line numbers - Consists of all tag sets that define the tool's input parameters. Each <param> tag within the <inputs> tag set maps to a command line parameter within the 'command' tag set described above. + Consists of all tag sets that define the +tool's input parameters. Each <param> tag within the <inputs> tag set +maps to a command line parameter within the [``command``](#tool|command) tag. Most +tools will not need to specify any attributes on this tag itself. + - Documentation for action + URL used by data source tools. - + - Documentation for check_values + Set to ``false`` to disable parameter checking in data source tools. - Documentation for method + Data source HTTP action (e.g. ``get`` or ``put``) to use. - Documentation for target + UI link target to use for data source tools (e.g. ``_top``). - + - Boolean indicates if this is an upload tool or not. + This boolean indicates if this is an upload tool or not. @@ -946,7 +950,59 @@ Is referenced parameter is the same group.]]> - This is a container for any tag sets that can be contained within the 'inputs' tag set. When this is used, the tool will allow the user to add any number of additional sets of the contained parameters ( an "Add new 'title'" button will be displayed on the tool form ). + `` tag set. When this is used, the +tool will allow the user to add any number of additional sets of the contained +parameters (an option to add new iterations will be displayed on the tool form). +All input parameters contained within the ```` tag can be retrieved by +enumerating over ``$`` in the relevant Cheetah code. +This returns the rank and the parameter objects of the repeat container. See the +Cheetah code below. + +### Example + +This is part is contained in the ```` tag set. + +```xml + + + + + +``` + +This Cheetah code can be used in the ```` tag set or the +```` tag set. + +```xml +#for $i, $s in enumerate( $series ) + rank_of_series=$i + input_path=${s.input.file_name} + x_colom=${s.xcol} + y_colom=${s.ycol} +#end for +``` + +### Testing + +This is an example test case with multiple repeat elements for the example above. Instances are created by referring to names with a special format: "_|" + +```xml + + + + + + + + +``` + +]]> @@ -984,9 +1040,33 @@ Is referenced parameter is the same group.]]> - - TODOWIKIMOVE - + +
+ +
+ +``` + +In your command template, you'll need to include the section name to access the +variable: + +``` +--color $adv.plot_color +``` + +Further examples can be found in the [test case](https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/section.xml) from [PR #35](https://github.com/galaxyproject/galaxy/pull/35) +]]> @@ -1364,26 +1444,31 @@ produces any standard error output). See [PR 117](https://github.com/galaxyproject/galaxy/pull/117) for more implementation information and discussion on the ``detect_errors`` attribute. +#### ``strict`` + +This boolean forces the ``#set -e`` directive on in shell scripts - so that in a +multi-part command if any part fails the job exits with a non-zero exit code. +This is enabled by default for tools with ``profile>=16.04`` and disabled on +legacy tools. + #### ``interpreter`` Older tools may define an ``intepreter`` attribute on the command, but this is deprecated and using the ``$__tool_directory__`` variable is superior. + +### Best Practices + +From https://galaxy-iuc-standards.readthedocs.org/en/latest/: + +The ``command`` tag should be started and finished by a [CDATA tag](https://en.wikipedia.org/wiki/CDATA). + ]]> - =16.04``, the default is ``exit_code`` - legacy tools default to ``default`` described above. -]]> - + From 2e31bc932dd9bd96d403cd8e6ef20715785c3eef Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 11 Sep 2016 15:46:16 -0400 Subject: [PATCH 080/109] Work on docs. --- galaxy.xsd | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 3bc0e29ef88c..c191d2387d23 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -890,13 +890,91 @@ tools will not need to specify any attributes on this tag itself. - This is a container for conditional parameters in the tool ( must contain 'when' tag sets ) - the command line is wrapped in an if-else statement. + + + + + + + + + + + + + + + + + + +``` + +The first directive following the conditional is a [``param``](#tool|inputs|param), +this param must be of type ``select`` or ``boolean``. Depending on the value a +user selects for this "test" parameter - different UI elements will be shown. +These different paths are described by the following the ``when`` blocks shown +above. + +The following Cheetah block demonstrates the use of the ``conditoinal`` +shown above: + +``` +biom convert -i "${input_type.input_table}" -o "${output_table}" +#if str( $input_type.input_type_selector ) == "tsv": + #if $input_type.process_obs_metadata: + --process-obs-metadata "${input_type.process_obs_metadata}" + #end if +#end if +``` + +Notice that the parameter ``input_table`` appears down both ``when`` clauses +so ``${input_type.input_table}`` appears unconditionally but we need to +conditionally reference ``${input_type.process_obs_metadata}`` with a Cheetah +``if`` statement. + +A common use of the conditional wrapper is to select between reference data +(managed by the Galaxy deployer or admin - for instance via Data Managers) or +history files for references. A good example tool that demonstrates this is +the [Bowtie 2](https://github.com/galaxyproject/tools-devteam/blob/master/tools/bowtie2/bowtie2_wrapper.xml) wrapper. + +```xml + + + + + + + + + + + + + + + + + +``` + +The Bowtie 2 wrapper also demonstrates other conditional paths - such as choosing +between paired inputs of single stranded inputs. +]]> - + @@ -934,14 +1012,18 @@ Is referenced parameter is the same group.]]> - Documentation for ConditionalWhen + This directive describes one potential +set of input for the tool at this depth. See documentation for the +[``conditional``](#tool|conditional) block for more details and examples (XML +and corresponding Cheetah conditionals). - Documentation for value + Value for the tool form test parameter +corresponding to this ``when`` block. @@ -951,7 +1033,6 @@ Is referenced parameter is the same group.]]> `` tag set or the ```xml #for $i, $s in enumerate( $series ) rank_of_series=$i - input_path=${s.input.file_name} + input_path=${s.input} x_colom=${s.xcol} y_colom=${s.ycol} #end for From 8f2f62a75a7aa256904821888c09682ebde99c83 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 11 Sep 2016 23:04:34 -0400 Subject: [PATCH 081/109] More docs... - Added docs for param's option. - Started work on validators and options, but are WIP though. --- galaxy.xsd | 155 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 150 insertions(+), 5 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index c191d2387d23..8d4a2a9addce 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1569,18 +1569,54 @@ The ``command`` tag should be started and finished by a [CDATA tag](https://en.w - Documentation for ParamOption + ``). + +### Example + +```xml + + + + + +``` + +An option can also be annotated with ``selected="true"`` to specify a +default option. + +```xml + + + + + + + +``` +]]> + - Documentation for value + - Documentation for selected + A boolean parameter indicating +if the corresponding option is selected by default (the default is ``false``). + @@ -1590,7 +1626,59 @@ The ``command`` tag should be started and finished by a [CDATA tag](https://en.w - Documentation for Options + `` tag when the ``type`` attribute value is ``select`` or +``data`` and used to dynamically generated lists of options. This tag set +dynamically creates a list of options whose values can be +obtained from a predefined file stored locally or a dataset selected from the +current history. + +There are four basic ways to use this tag - each corresponding to an attribute +on the ``options`` directive. These are: + +* ``from_data_table`` - The options for the select list are dynamically obtained + from a file specified in the Galaxy configuration file + ``tool_data_table_conf.xml`` or from a Tool Shed installed data manager. +* ``from_dataset`` - The options for the select list are dynamically obtained + from input dataset selected for the tool from the current history. +* ``from_file`` - The options for the select list are dynamically obtained from + a file. This mechanis is discourage in favor of the more generic + ``from_data_table``. +* ``from_parameter`` - The options for the select list are dynamically obtained + from a parameter. + +### ``from_data_table`` + + +### ``from_dataset`` + +https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml + +### ``from_file`` + +The following example is for Blast databases. In this example users maybe select +a database that is pre-formatted and cached in Galaxy clusters. When a new +dataset is available, admins must add the database to the local file named +"blastdb.loc". All such databases in that file are included in the options of +the select list. For a local instance, the file (e.g. ``blastdb.loc`` or +``alignseq.loc``) must be stored in the configured +[``tool_data_path``](https://github.com/galaxyproject/galaxy/tree/master/tool-data) +directory. In this example, the option names and values are taken from column 0 +of the file. + +```xml + + + + + + +``` +]]> + @@ -1660,7 +1748,64 @@ The ``command`` tag should be started and finished by a [CDATA tag](https://en.w - Documentation for Validator + + tag set - it applies a validator to the containing parameter. + +### Examples + +The following demonstrates a simple validator ``unspecified_build`` ensuring +that a dbkey is present on the selected dataset. This example is taken from the +[extract_genomic_dna](https://github.com/galaxyproject/tools-iuc/blob/master/tools/extract_genomic_dna/extract_genomic_dna.xml#L42) +tool. + +```xml + + + +``` + +Along the same line, the following example taken from +[samtools_mpileup](https://github.com/galaxyproject/tools-devteam/blob/master/tool_collections/samtools/samtools_mpileup/samtools_mpileup.xml) +ensures that a dbkey is present and that FASTA indices in the ``fasta_indexes`` +tool data table are present. + +```xml + + + + +``` + +In this older, somewhat deprecated example - a genome build of the dataset must +be stored in Galaxy clusters and the name of the genome (``dbkey``) must be one +of the values in the first column of file ``alignseq.loc`` - that could be +expressed with the validator. In general, ``dataset_metadata_in_file`` should be +considered deprecated in favor of + +```xml + +``` + +A very common validator is simply ensure a Python expression is valid for a +specified value. In the following example - paths/names that downstream tools +use in filenames may not contain ``..``. + +```xml +'..' not in value +``` +]]> From 4bcfe568b713e9369d0d1e5090e140d7ebfee17c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 00:07:39 -0400 Subject: [PATCH 082/109] More docs for validators. --- galaxy.xsd | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 8d4a2a9addce..0b6247d18db1 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1811,62 +1811,80 @@ use in filenames may not contain ``..``. - Documentation for type + - Documentation for message + +The error message displayed on the tool form if validation fails. - Documentation for check + Comma-seperated list of metadata +fields to check for if type is ``metadata``. If not specified, all non-optional +metadata fields will be checked unless they appear in the list of fields +specified by the ``skip`` attribute. - Documentation for filename + Tool data filename to check against +if ``type`` is ``dataset_metadata_in_file``. File should be present Galaxy's +``tool-data`` directory. - Documentation for metadata_name + Target metadata attribute name for +``dataset_metadata_in_data_table`` and ``dataset_metadata_in_file`` options. - This can be an integer index to the column or a column name. + Target column for metadata attribute +in ``dataset_metadata_in_data_table`` and ``dataset_metadata_in_file`` options. +This can be an integer index to the column or a column name. - Documentation for line_startswith + Used to indicate lines in the file +being used for validation start with a this attribute value. - Documentation for min + When the ``type`` attribute value is +``in_range`` - this is the minimum number allowed. - Documentation for max + When the ``type`` attribute value is +``in_range`` - this is the maximum number allowed. - Documentation for split + - Documentation for skip + - Documentation for table_name + From 2980835dc7ec8b2ba2e1cde39142e0697c4fc9f8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 11:49:03 -0400 Subject: [PATCH 083/109] A lot more documentation around parameter options. Also fixed a bit of a problem - turns out in action conditionals is very different than filter in options so I've split these types out. --- galaxy.xsd | 349 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 312 insertions(+), 37 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 0b6247d18db1..3f9ba0f84e44 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1636,19 +1636,21 @@ dynamically creates a list of options whose values can be obtained from a predefined file stored locally or a dataset selected from the current history. -There are four basic ways to use this tag - each corresponding to an attribute -on the ``options`` directive. These are: +There are at least five basic ways to use this tag - for of these correspond to +a ``from_XXX`` attribute on the ``options`` directive and the other is to just +use filters. +* Using a filter. * ``from_data_table`` - The options for the select list are dynamically obtained - from a file specified in the Galaxy configuration file - ``tool_data_table_conf.xml`` or from a Tool Shed installed data manager. + from a file specified in the Galaxy configuration file + ``tool_data_table_conf.xml`` or from a Tool Shed installed data manager. * ``from_dataset`` - The options for the select list are dynamically obtained - from input dataset selected for the tool from the current history. + from input dataset selected for the tool from the current history. * ``from_file`` - The options for the select list are dynamically obtained from - a file. This mechanis is discourage in favor of the more generic - ``from_data_table``. + a file. This mechanis is discourage in favor of the more generic + ``from_data_table``. * ``from_parameter`` - The options for the select list are dynamically obtained - from a parameter. + from a parameter. ### ``from_data_table`` @@ -1733,16 +1735,48 @@ of the file. - Documentation for Column + `` tag set - specifies columns used in building select options from a +file stored locally (i.e. index or tool data) or a dataset in the +current history. + +Any number of columns may be described, but at least one must be given the name +``value`` and it will serve as the value of this parameter in the Cheetah +template and elsewhwere (e.g. in API for instance). + +If a column named ``name`` is defined, this too has special meaning and it will +be the value the tool form user sees for each option. If no ``name`` column +appears, ``value`` will serve as the name. + +### Examples + +The following fragment shows options from the dataset in the current history +that has been selected as the value of the parameter named ``input1``. + +```xml + + + + +``` + +The [interval2maf](https://github.com/galaxyproject/galaxy/blob/dev/tools/maf/interval2maf.xml) +tool makes use of this tag with files from a history, and the +[star_fusion](https://github.com/galaxyproject/tools-iuc/blob/master/tools/star_fusion/star_fusion.xml) +tool makes use of this to reference a data table. +]]> - Documentation for name + Name given to the column with index +``index``, the names ``name`` and ``value`` have special meaning as described +above. - Documentation for index + 0-based index of the column in the +target file. @@ -1834,6 +1868,15 @@ metadata fields will be checked unless they appear in the list of fields specified by the ``skip`` attribute. + + + Tool data table name to check against +if ``type`` is ``dataset_metadata_in_tool_data``. See the documentation for +[tool data tables](https://wiki.galaxyproject.org/Admin/Tools/Data%20Tables) +and [data managers](https://wiki.galaxyproject.org/Admin/Tools/DataManagers) for +more information. + + Tool data filename to check against @@ -1872,22 +1915,33 @@ being used for validation start with a this attribute value. ``in_range`` - this is the maximum number allowed. - + - + When the ``type`` attribute value is +``in_range`` - this boolean indicates if the ``min`` value is allowed. - + - + When the ``type`` attribute value is +``in_range`` - this boolean indicates if the ``max`` value is allowed. - + - + If ``type`` is `dataset_metadata_in_file``, +this attribute is the column separator to use for values in the specified file. +This default is ``\t`` and due to a bug in older versions of Galaxy, should +not be modified. + + + + + Comma-seperated list of metadata +fields to skip if type is ``metadata``. If not specified, all non-optional +metadata fields will be checked unless ``check`` attribute is specified. - @@ -2034,61 +2088,187 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - Documentation for Filter + `` tag set - filter out values obtained from a locally stored file (e.g. +a tool data table) or a dataset in the current history. + +### Examples + +The following example from Mothur's +[remove.groups.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.groups.xml) +tool demonstrates filtering a select list based on the metadata of an input to +to the tool. + +```xml + + + + + + +``` + +This more advanced example, taken from Mothur's +[remove.linage.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml) +tool demonstrates using filters to sort a list and remove duplicate entries. + +```xml + + + + + + + + + + + + + + + + + +``` + +This example taken from the +[hisat2](https://github.com/galaxyproject/tools-iuc/blob/master/tools/hisat2/hisat2.xml) +tool demonstrates filtering values from a tool data table. + +```xml + + + + + + +``` + +The +[gemini_load.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/gemini/gemini_load.xml) +tool demonstrates adding values to an option list using ``filter``s. + +```xml + + + + + + + +``` + +While this fragment from maf_to_interval.xml demonstrates removing items. + +```xml + + + + + + +``` + +This example taken from +[snpSift_dbnsfp.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tool_collections/snpsift/snpsift_dbnsfp/snpSift_dbnsfp.xml) +demonstrates splitting up strings into multiple values. + +```xml + + + + + + + + +``` +]]> - Documentation for type + - + - Documentation for name + Column targeted by this filter - this +attribute is unused and invalid if ``type`` is ``add_value`` or ``remove_value``. +This can be a column index or a column name. - + - This can be a column index or a column name. + Name displayed for value to add (only +used with ``type`` of ``add_value``). - Documentation for ref + The attribute name of the reference file +(tool data) or input dataset. Only used when ``type`` is +``data_meta`` (required), ``param_value`` (required), or ``remove_value`` +(optional). - Documentation for key + When ``type`` is ``data_meta``, ``param_value``, +or ``remove_value`` - this is the name of the metadata key of ref to filter by. - + - Documentation for separator + - + - Documentation for compare + - + - Documentation for keep + If ``true``, keep columns matching the +value, if ``false`` discard columns matching the value. Used when ``type`` is +either ``static_value`` or ``param_value``. - Documentation for value + Target value of the operations - has +slightly different meanings depending on ``type``. For instance when ``type`` is +``add_value`` it is the value to add to the list and when ``type`` is +``static_value`` it is the value compared against. - Documentation for ref_attribute + Only used when ``type`` is +``param_value``. Period (``.``) separated attribute chain of input (``ref``) +attributes to use as value for filter. - Documentation for index + Used when ``type`` is ``add_value``, it +is the index into the list to add the option to. If not set, the option will be +added to the end of the list. @@ -2451,7 +2631,86 @@ Example: - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -3091,7 +3350,7 @@ Then whenever the user selects the "interval"" option from the select list, the - Documentation for FilterType + @@ -3103,6 +3362,21 @@ Then whenever the user selects the "interval"" option from the select list, the + + + + + + + + + + + + + + + Documentation for ActionType @@ -3128,6 +3402,7 @@ Then whenever the user selects the "interval"" option from the select list, the + From cd5328dce2aff381df95fa7b523dc40bc525f704 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 15:14:14 -0400 Subject: [PATCH 084/109] More documentation... Cleaned up the stdio tags and help section, flushed out citations and parameter options. Small touchups other places also. --- galaxy.xsd | 290 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 234 insertions(+), 56 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 3f9ba0f84e44..4c3f851c8f9e 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -91,18 +91,19 @@ substitution. + + + + + + + + + + + + + + + + +``` + +A minimal example wouldn't even need the ``filter`` or ``validator`` above, but +they are frequently nice features to add to your wrapper and can improve the user +experience of a tool. ### ``from_dataset`` -https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml +The following example is taken from the Mothur tool +[remove.lineage.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml) +and demonstrates generating options from a dataset directly. + +```xml + + + + + + + + + + + + + + + + + +``` + +Filters can be used to generate options from dataset directly also as the +example below demonstrates (many more examples are present in the +[``filter``](#tool|inputs|param|options|filter) documentation). + +```xml + + + + + +``` ### ``from_file`` @@ -1679,6 +1750,17 @@ of the file. ``` + +In general, ``from_file`` should be considered deprecated and ``from_data_table`` +should be prefered. + +### ``from_parameter`` + +This variant of the ``options`` directive is discouraged because it exposes +internal Galaxy structures. See the older +[bowtie](https://github.com/galaxyproject/tools-devteam/blob/master/tools/bowtie_wrappers/bowtie_wrapper.xml) +wrappers for an example of these. + ]]> @@ -3018,11 +3100,30 @@ The target value (e.g. for setting data format: the list of supported data forma `` tag has two subtags, ```` and ````, to define regular expressions and exit code processing, respectively. They are defined below. If a tool does not have any valid ```` or ```` tags, then Galaxy will use the previous technique for finding errors. +Legacy tools (ones with ``profile`` unspecified or a ``profile`` of less than +16.04) will default to checking stderr for errors as described above. Newer +tools will instead treat an exit code other than 0 as an error. The +``detect_error`` on ``command`` can swap between these behaviors but the +``stdio`` directive allows more options in defining error conditions (though +these aren't always intuitive). + +With ``stdio`` directive, Galaxy can use regular expressions to scan stdout and +stderr, and it also allows exit codes to be scanned for ranges. The ```` +tag has two subtags, ```` and ````, to define regular +expressions and exit code processing, respectively. They are defined below. If a +tool does not have any valid ```` or ```` tags, then Galaxy +will use the previous technique for finding errors. + +A note should be made on the order in which exit codes and regular expressions +are applied and how the processing stops. Exit code rules are applied before +regular expression rules. The rationale is that exit codes are more clearly +defined and are easier to check computationally, so they are applied first. Exit +code rules are applied in the order in which they appear in the tool's +configuration file, and regular expressions are also applied in the order in +which they appear in the tool's configuration file. However, once a rule is +triggered that causes a fatal error, no further rules are +checked.]]> -A note should be made on the order in which exit codes and regular expressions are applied and how the processing stops. Exit code rules are applied before regular expression rules. The rationale is that exit codes are more clearly defined and are easier to check computationally, so they are applied first. Exit code rules are applied in the order in which they appear in the tool's configuration file, and regular expressions are also applied in the order in which they appear in the tool's configuration file. However, once a rule is triggered that causes a fatal error, no further rules are checked.]]> @@ -3047,15 +3148,16 @@ Note that most Unix and Linux variants only support positive integers 0 to 255 f The tag's supported attributes are as follows: -* range: This indicates the range of exit codes to check. The range can be one of the following: - * n: the exit code will only be compared to n; - * [m:n]: the exit code must be greater than or equal to m and less than or equal to n; - * [m:]: the exit code must be greater than or equal to m; - * [:n]: the exit code must be less than or equal to n. -* level: This indicates the error level of the exit code. The level can have one of two values: - * warning: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A warning-level error will not cause the tool to fail. - * fatal: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. -* description: This is an optional description of the error that corresponds to the exit code. +* ``range``: This indicates the range of exit codes to check. The range can be one of the following: + * ``n``: the exit code will only be compared to n; + * ``[m:n]``: the exit code must be greater than or equal to m and less than or equal to n; + * ``[m:]``: the exit code must be greater than or equal to m; + * ``[:n]``: the exit code must be less than or equal to n. +* ``level``: This indicates the error level of the exit code. The level can have one of two values: + * ``warning``: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A warning-level error will not cause the tool to fail. + * ``fatal``: If an exit code falls in the given range, then a description of the error will be added to the beginning of stderr. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. +* ``description``: This is an optional description of the error that corresponds to the exit code. + The following is an example of the tag: ```xml @@ -3066,7 +3168,13 @@ The following is an example of the tag: ``` -If the tool returns 0 or 1, then the tool will not be marked as having an error. If the exit code is 2, then the tool will fail with the description "Out of Memory" added to stderr. If the tool returns 3, 4, or 5, then the tool will not be marked as having failed, but "Low disk space" will be added to stderr. Finally, if the tool returns any number greater than or equal to 6, then the description "Bad input dataset" will be added to stderr and the tool will be marked as having failed. +If the tool returns 0 or 1, then the tool will not be marked as having an error. +If the exit code is 2, then the tool will fail with the description ``Out of +Memory`` added to stderr. If the tool returns 3, 4, or 5, then the tool will not +be marked as having failed, but ``Low disk space`` will be added to stderr. +Finally, if the tool returns any number greater than or equal to 6, then the +description ``Bad input dataset`` will be added to stderr and the tool will be +marked as having failed. ]]> @@ -3093,26 +3201,28 @@ If the tool returns 0 or 1, then the tool will not be marked as having an error. tag does not contain the match attribute, then the tag will be ignored. The regular expression can be any valid Python regular expression. All regular expressions are performed case insensitively. For example, if match contains the regular expression "actg", then the regular expression will match against "actg", "ACTG", "AcTg", and so on. Also note that, if double quotes (") are to be used in the match attribute, then the value " can be used in place of double quotes. Likewise, if single quotes (') are to be used in the match attribute, then the value ' can be used if necessary. -* level: This works very similarly to the tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. For example, if stdout matches on a regular expression, then the regular expression's description is added to the beginning of stdout (instead of stderr). The level can be log, warning or fatal as described below. - * log and warning: If the regular expression matches against its source input (i.e., stdout and/or stderr), then a description of the error will be added to the beginning of the source, prepended with either 'Log:' or 'Warning:'. A log-level/warning-level error will not cause the tool to fail. - * fatal: If the regular expression matches against its source input, then a description of the error will be added to the beginning of the source. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. -* description: Just like its counterpart, this is an optional description of the regular expression that has matched. +* ``source``: This tells whether the regular expression should be matched against stdout, stderr, or both. If this attribute is missing or is incorrect, then both stdout and stderr will be checked. The source can be one of the follwing values: + * ``stdout``: the regular expression will be applied to stdout; + * ``stderr``: the regular expression will be applied to stderr; + * ``both``: the regular expression will be applied to both stderr and stdout (which is the default case). +* ``match``: This is the regular expression that will be used to match against stdout and/or stderr. If the tag does not contain the match attribute, then the tag will be ignored. The regular expression can be any valid Python regular expression. All regular expressions are performed case insensitively. For example, if match contains the regular expression "actg", then the regular expression will match against "actg", "ACTG", "AcTg", and so on. Also note that, if double quotes (") are to be used in the match attribute, then the value " can be used in place of double quotes. Likewise, if single quotes (') are to be used in the match attribute, then the value ' can be used if necessary. +* ``level``: This works very similarly to the tag, except that, when a regular expression matches against its source, the description is added to the beginning of the source. For example, if stdout matches on a regular expression, then the regular expression's description is added to the beginning of stdout (instead of stderr). The level can be log, warning or fatal as described below. + * ``log`` and ``warning``: If the regular expression matches against its source input (i.e., stdout and/or stderr), then a description of the error will be added to the beginning of the source, prepended with either 'Log:' or 'Warning:'. A log-level/warning-level error will not cause the tool to fail. + * ``fatal``: If the regular expression matches against its source input, then a description of the error will be added to the beginning of the source. A fatal-level error will cause the tool to fail. If no level is specified, then the fatal error level will be assumed to have occurred. +* ``description``: Just like its ``exit_code`` counterpart, this is an optional description of the regular expression that has matched. The following is an example of regular expressions that may be used: @@ -3134,13 +3244,37 @@ The following is an example of regular expressions that may be used: ``` -The regular expression matching proceeds as follows. First, if either stdout or stderr match on "low space", then a warning is registered. If stdout contained the string "---LOW SPACE---", then stdout has the string "Warning: Low space on device" added to its beginning. The same goes for if stderr had contained the string "low space". Since only a warning could have occurred, the processing continues. - -Next, the regular expression "error" is matched only against stdout. If stdout contains the string "error" regardless of its capitalization, then a fatal error has occurred and the processing stops. In that case, stdout would be prepended with the string "Fatal: Unknown error encountered". Note that, if stderr contained "error", "ERROR", or "ErRor" then it would not matter - stderr was not being scanned. +The regular expression matching proceeds as follows. First, if either stdout or +stderr match on ``low space``, then a warning is registered. If stdout contained +the string ``---LOW SPACE---``, then stdout has the string ``Warning: Low space +on device`` added to its beginning. The same goes for if stderr had contained the +string ``low space``. Since only a warning could have occurred, the processing +continues. + +Next, the regular expression ``error`` is matched only against stdout. If stdout +contains the string ``error`` regardless of its capitalization, then a fatal +error has occurred and the processing stops. In that case, stdout would be +prepended with the string ``Fatal: Unknown error encountered``. Note that, if +stderr contained ``error``, ``ERROR``, or ``ErRor`` then it would not matter - +stderr was not being scanned. + +If the second regular expression did not match, then the third regular +expression is checked. The third regular expression does not contain an error +level, so an error level of ``fatal`` is assumed. The third regular expression +also does not contain a source, so both stdout and stderr are checked. The third +regular expression looks for 12 consecutive "C"s or "G"s in any order and in +uppercase or lowercase. If stdout contained ``cgccGGCCcGGcG`` or stderr +contained ``CCCCCCgggGGG``, then the regular expression would match, the tool +would be marked with a fatal error, and the stream that contained the +12-nucleotide CG island would be prepended with ``Fatal: Fatal error - CG island +12 nts long found``. + +Finally, if the tool did not match any of the fatal errors, then the fourth +regular expression is checked. Since no source is specified, both stdout and +stderr are checked. If ``Branch A`` is at the beginning of stdout or stderr, then +a warning will be registered and the source that contained ``Branch A`` will be +prepended with the warning ``Warning: Branch A was taken in execution``. -If the second regular expression did not match, then the third regular expression is checked. The third regular expression does not contain an error level, so an error level of "fatal" is assumed. The third regular expression also does not contain a source, so both stdout and stderr are checked. The third regular expression looks for 12 consecutive "C"s or "G"s in any order and in uppercase or lowercase. If stdout contained "cgccGGCCcGGcG" or stderr contained "CCCCCCgggGGG", then the regular expression would match, the tool would be marked with a fatal error, and the stream that contained the 12-nucleotide CG island would be prepended with "Fatal: Fatal error - CG island 12 nts long found". - -Finally, if the tool did not match any of the fatal errors, then the fourth regular expression is checked. Since no source is specified, both stdout and stderr are checked. If "Branch A" is at the beginning of stdout or stderr, then a warning will be registered and the source that contained "Branch A" will be prepended with the warning "Warning: Branch A was taken in execution". ]]> @@ -3223,8 +3357,49 @@ Then whenever the user selects the "interval"" option from the select list, the - Tool files may declare one citations element. Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. - + + + 10.1093/bioinformatics/btq281 + + + @ARTICLE{Kim07aninterior-point, + author = {Seung-jean Kim and Kwangmoo Koh and Michael Lustig and Stephen Boyd and Dimitry Gorinevsky}, + title = {An interior-point method for large-scale l1-regularized logistic regression}, + journal = {Journal of Machine Learning Research}, + year = {2007}, + volume = {8}, + pages = {1519-1555} + } + +``` + +For more implementation information see the +[pull request](https://bitbucket.org/galaxy/galaxy-central/pull-requests/440/initial-bibtex-doi-citation-support-in/diff) +adding this feature. For more examples of how to add this to tools checkout the +following commits adding this to the +[NCBI BLAST+ suite](https://github.com/peterjc/galaxy_blast/commit/9d2e3906915895765ecc3f48421b91fabf2ccd8b), +[phenotype association tools](https://bitbucket.org/galaxy/galaxy-central/commits/39c983151fe328ff5d415f6da81ce5b21a7e18a4), +[MAF suite](https://bitbucket.org/galaxy/galaxy-central/commits/60f63d6d4cb7b73286f3c747e8acaa475e4b6fa8), +and [MACS2 suite](https://github.com/jmchilton/galaxytools/commit/184971dea73e236f11e82b77adb5cab615b8391b). + +This feature was added to the August 2014 release of Galaxy, tools annotated +with citations will work in older releases of Galaxy but no citation information +will be available to the end user. +]]> @@ -3233,13 +3408,16 @@ Then whenever the user selects the "interval"" option from the select list, the - Each citations element can contain one or more citation tag elements - each of which specifies tool citation information using either a DOI or a BibTeX entry. + Each citations element can contain one or +more ``citation`` tag elements - each of which specifies tool citation +information using either a DOI or a BibTeX entry. - type of citation + Type of citation - currently ``doi`` +and ``bibtex`` are the only supported options. From 24f46245e09ef044cca6804c48c994e49b70b69a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 15:25:21 -0400 Subject: [PATCH 085/109] Mention code files as a way to generate dynamic options. Transplant example from Galaxy wiki of doing this. --- galaxy.xsd | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 4c3f851c8f9e..9b717b86a426 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -305,7 +305,99 @@ communicating with an external data source application (the default is ``get``). - **Deprecated** do not use this unless absolutely necessary. This tag set provides detailed control of the way the tool is executed. This (optional) code can be deployed in a separate file in the same directory as the tool's config file. These hooks are being replaced by new tool config features and methods in the ~/lib/galaxy/tools/__init__.py code file. + +``` + +Here are the relevant input parameters in our tool config. The first parameter +is the input dataset that includes the above metadata elements. + +```xml + + value is not None and len(value.metadata.field_names) > 0 + +``` + +The following parameter dynamically renders a select list consisting of the +elements in the ``field_names`` metadata element associated with the selected +input dataset. + +```xml + + + + + + +``` + +The following parameter calls the ``get_field_components_options()`` function in +the ``tool_form_utils.py`` code file discussed above. This function returns the +value of the input dataset's ``field_components`` metadata element dictionary +whose key is the currently selected ``field_name`` from the select list parameter +above. + +```xml + +``` + +Changing the selected option in the ``field_name`` select list will dynamically +re-render the options available in the associated ``field_component_index`` select +list, which is the behavior we want. + +The ``get_field_components_options()`` method looks like this. + +```python +def get_field_components_options( dataset, field_name ): + options = [] + if dataset.metadata is None: + return options + if not hasattr( dataset.metadata, 'field_names' ): + return options + if dataset.metadata.field_names is None: + return options + if field_name is None: + # The expression validator that helps populate the select list of input + # datsets in the icqsol_color_surface_field tool does not filter out + # datasets with no field field_names, so we need this check. + if len( dataset.metadata.field_names ) == 0: + return options + field_name = dataset.metadata.field_names[0] + field_components = dataset.metadata.field_components.get( field_name, [] ) + for i, field_component in enumerate( field_components ): + options.append( ( field_component, field_component, i == 0 ) ) + return options +``` + +]]> + @@ -1761,6 +1853,11 @@ internal Galaxy structures. See the older [bowtie](https://github.com/galaxyproject/tools-devteam/blob/master/tools/bowtie_wrappers/bowtie_wrapper.xml) wrappers for an example of these. +### Other Ways to Dynamically Generate Options + +Though deprecated and discouraged, [``code``](#tool|code) blocks can also be +used to generate dynamic options. + ]]> From 6481a00f88bff4bb7a7adbf2aaaee639e4649147 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 16:19:01 -0400 Subject: [PATCH 086/109] Flush out docs for requirements and child tags. --- galaxy.xsd | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 9b717b86a426..348aa87fad3b 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -230,7 +230,15 @@ communicating with an external data source application (the default is ``get``). - This is a container tag set for the 'requirement' tag set described below. + @@ -240,7 +248,53 @@ communicating with an external data source application (the default is ``get``). - This tag set is contained within the 'requirements' tag set. Third party programs or modules that the tool depends upon (and which are not distributed with Galaxy) are included in this tag set. The intention is that when Galaxy starts it can check whether the required programs or modules are available, and if not this tool will not be loaded. + + samtools + +``` + +This older example shows a tool that requires R version 2.15.1. The +``tool_depensencies.xml`` should contain matching declarations for Galaxy to +actually install the R runtime. The ``set_envirornment`` type is only respected +by the tool shed and is ignored by the newer and preferred conda dependency +resolver. + +```xml + + R_SCRIPT_PATH + R + +``` + +]]> @@ -259,8 +313,17 @@ communicating with an external data source application (the default is ``get``). - This tag set is contained within the 'requirements' tag set. Galaxy can be configured to run tools within Docker (https://www.docker.com/) containers - this tag allows the tool to suggest possible valid Docker containers for this tool. - + From 3af76de52ff56fca6b65d993ce91fb39c30af2d2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 16:55:28 -0400 Subject: [PATCH 087/109] Flush out docs for sanitizers, configfiles, etc... --- galaxy.xsd | 91 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 22 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 348aa87fad3b..461850633ddc 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -2190,9 +2190,20 @@ metadata fields will be checked unless ``check`` attribute is specified. tag set - it contains a set of and tags. -Examples: +See +[/tools/filters/grep.xml](https://github.com/galaxyproject/galaxy/blob/dev/tools/filters/grep.xml) +for a typical example of how to use this tag set. This tag set is used to +replace the basic parameter sanitization with custom directives. This tag set is +contained within the ```` tag set - it contains a set of ```` and +```` tags. + +### Examples + +This example replaces the invalid character default of ``X`` with the empty +string (so invalid characters are effectively dropped instead of replaced with +``X``) and indicates the only valid characters for this input are ASCII letters, +ASCII digits, and ``_``. ``` @@ -2202,6 +2213,9 @@ Examples: ``` +This example allows many more valid characters and specifies that ``&`` will just +be dropped from the input. + ``` @@ -2219,12 +2233,14 @@ Examples: - is this parameter sanitized (default is ``true``) + This boolean parameter determines if the +input is sanitized at all (the default is ``true``). - character to replace invalid characters with + The attribute specifies the character +used as a replacement for invalid characters. @@ -2236,21 +2252,25 @@ Examples: - tag set. Used to specify a list of allowed characters. Contains and tags.]]> + `` tag set, these are used to specify a list of allowed characters. +Contains ```` and ```` tags.]]> - initial characters to allow -no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` + This describes the initial characters to +allow as valid, the default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - Used to add individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). + This directive is used to add individual +characters or preset lists of characters. Character must not be allowed as a +valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). @@ -2265,7 +2285,10 @@ no, default is ``string.letters + string.digits + " -=_.()/+*^,:?!"`` - Used to remove individual characters or preset lists of characters. Character must not be allowed as a valid input for the mapping to occur. Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). + This directive is used to remove +individual characters or preset lists of characters. +Character must not be allowed as a valid input for the mapping to occur. +Preset lists include default and none as well as those available from string.* (e.g. ``string.printable``). @@ -2982,7 +3005,11 @@ Example: - tag sets - it defines an additional configuration section.]]> + `` and ```` tag sets - which can be used +to setup configuration files for use by tools.]]> @@ -2999,11 +3026,14 @@ Example: tag set. It allows for the creation of a temporary file for file-based parameter transfer. + +This tag set is contained within the ```` tag set. It allows for +the creation of a temporary file for file-based parameter transfer. *Example* -The following is taken from the ~/tools/plotting/xy_plot.xml tool config. +The following is taken from the [xy_plot.xml](https://github.com/galaxyproject/tools-devteam/blob/master/tools/xy_plot/xy_plot.xml) +tool config. ```xml @@ -3037,13 +3067,21 @@ The following is taken from the ~/tools/plotting/xy_plot.xml tool config. ``` + +This file is then used in the ``command`` block of the tool as follows: + +```xml +bash "$__tool_directory__/r_wrapper.sh" "$script_file" +``` + ]]> - Documentation for name + Cheetah variable used to reference +the path to the file created with this directive. @@ -3053,7 +3091,9 @@ The following is taken from the ~/tools/plotting/xy_plot.xml tool config. tag set. It tells Galaxy to write out a JSON representation of the tool parameters . + +This tag set is contained within the tag set. It tells Galaxy to +write out a JSON representation of the tool parameters . *Example* @@ -3061,25 +3101,32 @@ The following will create a cheetah variable that can be evaluated as ``$inputs` will contain the tool parameter inputs. ```xml - - - + + + ``` -The following will instead write the inputs to the tool's working directory with the specified name (i.e. ``inputs.json``). +The following will instead write the inputs to the tool's working directory with +the specified name (i.e. ``inputs.json``). ```xml - - - + + + ``` + +A contrived example of a tool that uses this is the test tool +[inputs_as_json.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/inputs_as_json.xml). ]]> - Cheetah variable to populate the path to the inputs JSON file created in response to this directive. + From b059c8b592b0f45dd669f8e9cbca714ed292ccac Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 21:25:37 -0400 Subject: [PATCH 088/109] Working my way through basic parameter documentation. --- galaxy.xsd | 227 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 202 insertions(+), 25 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 461850633ddc..ff39fa98ed34 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1036,11 +1036,6 @@ tools will not need to specify any attributes on this tag itself. Documentation for InputType - - - Name for this element - - @@ -1132,6 +1127,11 @@ between paired inputs of single stranded inputs. + + + Name for this element + + + + + Name for this element + + The title of the repeat section, which will be displayed on the tool form @@ -1327,62 +1332,213 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - Documentation for Param + `` tag set - each of these specifies a field that +will be displayed on the tool form. Ultimately, the values of these form fields +will be passed as the command line parameters to the tool's executable. + +### Common Attributes + +The attributes valid for this tag vary wildly based on the ``type`` of the +parameter being described. All the attributes for the ``param`` element are +documented below for completeness, but here are the common ones for each +type are as follows: + +$attribute_list:name,type,optional,label,help,argument + +### Parameter Types + +#### ``text`` + +When ``type="text"``, the parameter is free form text and appears as a text box +in the tool form. + +$attribute_list:value,size,area + +##### Examples + +Sometimes you need labels for data or graph axes, chart titles, etc. This can be +done using a text field. The following will create a text box 30 characters wide +with the default value of "V1". + +```xml + +``` + +The ``size`` parameter can be two dimensional, if it is the textbox will be +rendered on the tool form as a text area instead of a single line text box. + +```xml + +``` + +#### ``integer`` and ``float`` + +These parameters represent whole number and real numbers, respectively. + +$attribute_list:value,min,max + +##### Example + +``` + +``` + +#### ``boolean`` + +This represents a binary true or false value. + +$attribute_list:checked,truevalue,falsevalue + +#### ``data`` + +A dataset from the current history. Multiple types might be used for the param form. + +##### Examples + +The following will find all "coordinate interval files" contained within the +current history and dynamically populate a select list with them. If they are +selected, their destination and internal file name will be passed to the +appropriate command line variable. + +```xml + +``` + +The following demonstrates a ``param`` which may accept multiple files and +multiple formats. + +```xml + +``` + +$attribute_list:format,multiple + +#### ``select`` + +The following will create a select list containing the options "Downstream" and +"Upstream". Depending on the selection, a ``d`` or ``u`` value will be passed to +the ``$upstream_or_down`` variable on the command line. + +```xml + + + + +``` + +The following will create a checkbox list allowing the user to select +"Downstream", "Upstream", both, or neither. Depending on the selection, the +value of ``$upstream_or_down`` will be ``d``, ``u``, ``u,d``, or "". + +```xml + + + + +``` + +$attribute_list:data_ref,dynamic_options,display,multiple + +]]> - - Documentation for type + + + + + + + - Documentation for type + Boolean indicating if this should be +rendered as a one line text box (if ``false``) or a multi-line text area (if +``true``). - Command-line argument this parameter maps to (e.g. ``--foobar``). + - Documentation for label + The attribute value will be displayed on the tool page as the label of the form field (``label="Sort Query"``). - Documentation for help + Short bit of text, rendered on the +tool form just below the associated field to provide information about the +field. - Documentation for value + The default value for this +parameter. - Documentation for optional + If ``false``, parameter must have a +value. Defaults to "false". - Documentation for min + Minimum valid parameter value - only +valid when ``type`` is ``integer`` or ``float``. - Documentation for max + Maximum valid parameter value - only +valid when ``type`` is ``integer`` or ``float``. - Documentation for format + Only if ``type`` attribute value is +``data`` or ``data_collection`` - the list of supported data formats is +contained in the +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) +file. Use the file extension. @@ -1392,7 +1548,14 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - Documentation for data_ref + @@ -1412,12 +1575,15 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - Documentation for display + This attribute is used only if +``type`` attribute value is ``select`` - render a select list as a set of check +boxes or radio buttons. Defaults to a drop-down menu select list. - Documentation for multiple + Allow multiple valus to be selected. +Valid with ``data`` and ``select`` parameters. @@ -1432,23 +1598,31 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - Documentation for checked + Set to ``true`` if the ``boolean`` +parameter should be checked (or ``true``) by default. - Documentation for truevalue + The parameter value in the Cheetah +template if the parameter is ``true`` or checked by the user. Only valid if +``type`` is ``boolean``. - Documentation for falsevalue + The parameter value in the Cheetah +template if the parameter is ``false`` or not checked by the user. Only valid if +``type`` is ``boolean``. - Documentation for size + Used only if ``type`` attribute +value is ``text``. To create a multi-line text box add an ``area="True"`` +attribute to the param tag. This can be one dimensional (e.g. ``size="40"``) +or two dimensional (e.g. ``size="5x25"``). @@ -1458,7 +1632,10 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - Deprecated method to allow access to Python code to generate options for a select list. + Deprecated/discouraged method to +allow access to Python code to generate options for a select list. See +[``code``'s](#tool|code) documentation for an example. + From 366def7026a68e2bde75a140e3e4f1496966b1f2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 22:17:40 -0400 Subject: [PATCH 089/109] More work on parameters and environment variables. --- galaxy.xsd | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 6 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index ff39fa98ed34..86438e40dcb5 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1170,7 +1170,7 @@ Is referenced parameter is the same group.]]> This directive describes one potential set of input for the tool at this depth. See documentation for the -[``conditional``](#tool|conditional) block for more details and examples (XML +[``conditional``](#tool|inputs|conditional) block for more details and examples (XML and corresponding Cheetah conditionals). @@ -1443,6 +1443,56 @@ value of ``$upstream_or_down`` will be ``d``, ``u``, ``u,d``, or "". $attribute_list:data_ref,dynamic_options,display,multiple +#### ``data_column`` + +$attribute_list:force_select,numerical,use_header_name + +#### ``drill_down`` + +$attribute_list:hierarchy + +#### ``data_collection`` + +$attribute_list:format,collection_type + +The following will create a parameter that only accepts paired FASTQ files. + +##### Examples + +```xml + + +``` + +More detailed information on writing tools that consume collections can be found +in the [planemo documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#collections). + +#### ``color`` + +$attribute_list:value + +##### Examples + +The following example will create a color selector parameter. + +```xml + + +``` + +Given that the output includes a pound sign, it is often convenient to use a +sanitizer to prevent Galaxy from escaping the result. + +```xml + + + + + + + +``` + ]]> @@ -1460,7 +1510,7 @@ the tool form widget is different. Currently valid parameter types are: ``color``, ``data_column``, ``hidden``, ``hidden_data``, ``baseurl``, ``file``, ``ftpfile``, ``data``, ``data_collection``, ``library_data``, ``drill_down``. The definition of supported parameter types as defined in the -``parameter_types dictionary`` in +``parameter_types`` dictionary in [/lib/galaxy/tools/parameters/basic.py](https://github.com/galaxyproject/galaxy/blob/master/lib/galaxy/tools/parameters/basic.py). ]]> @@ -1498,7 +1548,9 @@ attribute by stripping the dashes (e.g. if ``argument="--sensitive"`` then - The attribute value will be displayed on the tool page as the label of the form field (``label="Sort Query"``). + The attribute value will be +displayed on the tool page as the label of the form field +(``label="Sort Query"``). @@ -1543,7 +1595,14 @@ file. Use the file extension. - Restrict valid collection types for data_collection params (e.g. list, paired, list:paired, etc...). + @@ -1634,7 +1693,7 @@ or two dimensional (e.g. ``size="5x25"``). Deprecated/discouraged method to allow access to Python code to generate options for a select list. See -[``code``'s](#tool|code) documentation for an example. +``code``'s documentation for an example. @@ -3156,6 +3215,7 @@ Example: +This directive should contain one or more ``environment_variable`` definition. @@ -3166,13 +3226,47 @@ Example: + $inttest + #if int($inttest) == 3 +ISTHREE +#else# +NOTTHREE +#end if# + + +``` + +If these environment variables are used in another Cheetah context, such as in +the ``command`` block, the ``$`` used indicate shell expansion of a variable +should be escaped with a ``\`` so prevent it from being evaluated as a Cheetah +variable instead of shell variable. + +```xml + + echo "\$INTVAR" > $out_file1; + echo "\$IFTEST" >> $out_file1; + +``` + ]]> - Documentation for name + Name of the environment variable to +define. From 4ba421e2f67de03dfa51ad8493f9a27beb159f27 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 12 Sep 2016 23:48:22 -0400 Subject: [PATCH 090/109] Finishing first pass at filling out wiki docs for ``param``. And other small fixes. --- galaxy.xsd | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 86438e40dcb5..65416a206af3 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1445,6 +1445,8 @@ $attribute_list:data_ref,dynamic_options,display,multiple #### ``data_column`` +This parameter type is used to select columns from a parameter. + $attribute_list:force_select,numerical,use_header_name #### ``drill_down`` @@ -1493,6 +1495,9 @@ sanitizer to prevent Galaxy from escaping the result. ``` +This covers examples of the most common parameter types, the remaining parameter +types are more obsecure and less likely to be useful for most tool authors. + ]]> @@ -1617,19 +1622,25 @@ but not always the tool's input dataset). ]]> - + - Documentation for accept_default + - Documentation for force_select + Used only if the ``type`` attribute +value is ``data_column``, this is deprecated and the inverse of ``optional``. +Set to ``false`` to not force user to select an option in the list. - Documentation for use_header_names + >Used only if the ``type`` attribute +value is ``data_column``, if ``true`` Galaxy assumes first row of ``data_ref`` +is a header and builds the select list with these values rather than the more +generic ``c1`` ... ``cN``. + @@ -1647,12 +1658,16 @@ Valid with ``data`` and ``select`` parameters. - Documentation for numerical + Used only if the ``type`` attribute +value is ``data_column``, if ``true`` the column will be treated as numerical +when filtering columns based on metadata. - Documentation for hierarchy + Used only if the ``type`` attribute +value is ``drill_down``, this attribute determines the drill down is +``recursive`` or ``exact``. @@ -1684,11 +1699,17 @@ attribute to the param tag. This can be one dimensional (e.g. ``size="40"``) or two dimensional (e.g. ``size="5x25"``). + + Deprecated/discouraged method to From 5ad6e790e2f1a752beb748581469aa2e9f73477f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2016 09:48:18 -0400 Subject: [PATCH 091/109] Starting to polish and flush out data outputs documentation. --- galaxy.xsd | 182 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 159 insertions(+), 23 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 65416a206af3..babf370d1987 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -2797,7 +2797,12 @@ added to the end of the list. tag set. The files and collections created by tools as a result of their execution are named by Galaxy. You specify the number and type of your output files using the contained and tags. You must pass them to your tool executable through using line variables just like the parameters described in the previous sections. +Container tag set for the ```` and ```` tag sets. +The files and collections created by tools as a result of their execution are +named by Galaxy. You specify the number and type of your output files using the +contained ```` and ```` tags. These may be passed to your tool +executable through using line variables just like the parameters described in +the ```` documentation. ]]> @@ -2820,50 +2825,150 @@ Container tag set for the tag set. The files and collections created by t - TODOWIKIMOVE + `` tag set, and it defines the +output data description for the files resulting from the tool's execution. The +value of the attribute ``label`` can be acquired from input parameters or metadata +in the same way that the command line parameters are (discussed in the + tag set section above). + +### Examples + +The following will create a variable called ``$out_file1`` with data type +``pdf``. + + + + +The valid values for format can be found in +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample). + +The following will create a dataset in the history panel whose data type is the +same as that of the input dataset selected (and named ``input1``) for the tool. + +```xml + + + +``` + +The following will create datasets in the history panel, setting the output data +type to be the same as that of an input dataset named by the ``format_source`` +attribute. Note that a conditional name is not included, so 2 separate +conditional blocks should not contain parameters with the same name. + +```xml + + + + + + + + + + + + + + + + + + + qual['add'] == 'yes' + + +``` + +Assume that the tool includes an input parameter named ``database`` which is a +select list (as shown below). Also assume that the user selects the first option +in the ``$database`` select list. Then the following will ensure that the tool +produces a tabular data set whose associated history item has the label ``Blat +on Human (hg18)``. + +```xml + + + + + + + + + + +``` + +]]> - - - Documentation for name + + + - If ``true``, this output will sniffed and its format determined automatically by Galaxy. + - Documentation for format + The short name for the output datatype. +The valid values for format can be found in +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) +(e.g. ``format="pdf"`` or ``format="fastqsanger"``). - Documentation for format_source + This sets the data type of the output file to be the same format as that of a tool input dataset. - Documentation for metadata_source + This copies the metadata information +from the tool's input dataset. This is particularly useful for interval data +types where the order of the columns is not set. - Documentation for label + .}``, as +discussed for command line parameters in the ```` tag set section +above. The default label is ``${tool.name} on ${on_string}``. + +]]> - Documentation for from_work_dir + Relative path to a file produced by the +tool in its working directory. Output's contents are set to this file's +contents. - + - Documentation for hidden + Boolean indicating whether to hide +dataset in the history view. (Default is ``false``.) @@ -2924,7 +3029,15 @@ Container tag set for the tag set. The files and collections created by t `` tag can contain a ```` tag which includes a Python code block to be executed to test whether to include this output in the outputs the tool ultimately creates. If the code, when executed, returns ``True``, the output dataset is retained. In these code blocks the tool parameters appear as Python variables and are thus referred to without the $ used for the Cheetah template (used in the ```` tag). Variables that are part of conditionals are accessed using a hash named after the conditional. For example: +The ```` tag can contain a ```` tag which includes a Python code +block to be executed to test whether to include this output in the outputs the +tool ultimately creates. If the code, when executed, returns ``True``, +the output dataset is retained. In these code blocks the tool parameters appear +as Python variables and are thus referred to without the $ used for the Cheetah +template (used in the ```` tag). Variables that are part of +conditionals are accessed using a hash named after the conditional. + +### Example ```xml @@ -3584,7 +3697,8 @@ Example: @@ -3800,7 +3914,13 @@ prepended with the warning ``Warning: Branch A was taken in execution``. - tag set and is the container tag set for the following tag set.]]> + `` tag set and is the container tag set +for the following ```` tag set.]]> @@ -3809,12 +3929,14 @@ prepended with the warning ``Warning: Branch A was taken in execution``. @@ -3823,7 +3945,9 @@ Assume that your tool config includes the following select list parameter struct ``` -Then whenever the user selects the "interval"" option from the select list, the following structure in your tool config will override the format="fasta" setting in the tag set with format="interval". +Then whenever the user selects the ``interval`` option from the select list, the +following structure in your tool config will override the ``format="fasta"`` setting +in the ```` tag set with ``format="interval"``. ```xml @@ -3834,22 +3958,34 @@ Then whenever the user selects the "interval"" option from the select list, the ``` + +See +[extract_genomic_dna.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/extract_genomic_dna/extract_genomic_dna.xml) +or the test tool +[output_action_change_format.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/output_action_change_format.xml) +for more examples. + ]]> - This value must be the attribute name of the desired input parameter + This value must be the attribute name of +the desired input parameter (e.g. ``input="out_format"`` above). - This value must also be an attribute name of an input parameter + This value must also be an attribute +name of an input parameter (e.g. ``value="interval"`` above). - This value must be a supported data type (e.g. ``format="interval"``) + This value must be a supported data type +(e.g. ``format="interval"``). See +[/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample) +for a list of supported formats. From e2d55ccf1685ec767d2c09715b189c7a1ff411cb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2016 11:24:22 -0400 Subject: [PATCH 092/109] More docs - conditional output actions and some more on discover datasets. --- galaxy.xsd | 179 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 156 insertions(+), 23 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index babf370d1987..09ccb1472e7a 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -3073,10 +3073,19 @@ conditionals are accessed using a hash named after the conditional. - Describe datasets to dynamically collect after the job complete. + +]]> @@ -3111,7 +3120,20 @@ More information can be found at https://wiki.galaxyproject.org/Admin/Tools/Mult - See https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_list_of_pairs.xml and https://github.com/galaxyproject/galaxy/blob/master/test/functional/tools/collection_creates_dynamic_nested.xml for examples + @@ -3141,13 +3163,93 @@ More information can be found at https://wiki.galaxyproject.org/Admin/Tools/Mult - `` in the Bowtie wrapper is used in lieu of the deprecated tag to set the dbkey of the output dataset. In bowtie_wrapper.xml (see below), according to the first action block, if the refGenomeSource.genomeSource is "indexed" (not "history"), then it will assign the dbkey of the output file to be the same as that of the reference file. It does this by looking at through the loc file and finding the line that has the value that's been selected in the index dropdown box as column 1 of the loc file entry and using the dbkey, in column 0 (ignoring comment lines (starting with #) along the way). + tag) does then it should be changed to "equCab2" (the @@ -3187,24 +3288,37 @@ Example: - + - + - + Type of action (either ``format`` or +``metadata`` currently). - + If ``type="metadata"``, the name of the +metadata element. - + If ``type="format"``, the default format +if none of the nested options apply. @@ -3243,26 +3357,45 @@ Example: - + - - - - - - - + + + + + + Name of the input parameter to base +conditional logic on. The value of this parameter will be matched against nested +``when`` directives. + + - + - + Value to match conditional input value +against. From 89c8934727c7a8b8cc4ca7db10e7882d22542a87 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2016 14:51:55 -0400 Subject: [PATCH 093/109] A bunch of work on cleaning up and flushing out documentation for test tags. --- galaxy.xsd | 291 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 238 insertions(+), 53 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 09ccb1472e7a..3397b88de915 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -522,46 +522,63 @@ def get_field_components_options( dataset, field_name ): - tag sets. Any number of tests can be included, and each test is wrapped within separate tag sets. Functional tests are executed via Planemo or the ~/run_tests.sh shell script. + tag sets. Any number of tests can be included, +and each test is wrapped within separate tag sets. Functional tests are +executed via [Planemo](http://planemo.readthedocs.io/) or the +[run_tests.sh](https://github.com/galaxyproject/galaxy/blob/dev/run_tests.sh) +shell script distributed with Galaxy. -See [WritingTests](https://wiki.galaxyproject.org/Admin/Tools/WritingTests) on the wiki and [Test-Driven Development](http://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development) section of the Planemo docs for more information on writing tests.]]> +The documentation contained here is mostly reference documentation, for +tutorials on writing tool tests please check out Planemo's +[Test-Driven Development](http://planemo.readthedocs.io/en/latest/writing_advanced.html#test-driven-development) +documentation or the much older wiki content for +[WritingTests](https://wiki.galaxyproject.org/Admin/Tools/WritingTests). + +]]> + - - + - + - + - + -Example - -Test the execution of the MAF-to-FASTA converter ( ~/tools/maf/maf_to_fasta.xml ). +``` +The following example, tests the execution of the MAF-to-FASTA converter +([/tools/maf/maf_to_fasta.xml](https://github.com/galaxyproject/galaxy/blob/master/tools/maf/maf_to_fasta.xml)). -Toggle line numbers +```xml @@ -570,12 +587,13 @@ Toggle line numbers -Example - -This test demonstrates verifying specific properties about a test output instead of directly comparing it to another file. Here the file attribute is not specified and instead a series of assertions is made about the output. +``` +This test demonstrates verifying specific properties about a test output instead +of directly comparing it to another file. Here the file attribute is not +specified and instead a series of assertions is made about the output. -Toggle line numbers +```xml @@ -591,6 +609,8 @@ Toggle line numbers +``` + ]]> @@ -620,12 +640,15 @@ Toggle line numbers - Assert the number of outputs this test should produce. + Assert the number of outputs this test +should produce, this is useful to ensure ``filter`` directives are implemented correctly. - + - Expect the job to fail, if set to true no job output checks may be present in test definition. + Setting this to ``true`` indicates +the expectation is for the job fail. If set to ``true`` no job output checks may +be present in ``test`` definition. @@ -644,17 +667,27 @@ Toggle line numbers - Describe assertions about the jobs standard error. + Describe assertions about the job's +generated command-line. + +$assertions + - Describe assertions about the jobs standard error. + Describe assertions about the job's standard error. + +$assertions + - Describe assertions about the jobs standard error. + Describe assertions about the jobs standard error. + +$assertions + @@ -681,7 +714,13 @@ Toggle line numbers - This tag set defines the tool's input parameters for executing the tool via the functional test framework. + @@ -690,22 +729,29 @@ Toggle line numbers - This value must match the name of the associated input parameter. + This value must match the name of the +associated input parameter. - This value must be one of the legal values that can be assigned to an input parameter. + This value must be one of the legal +values that can be assigned to an input parameter. - This attribute name should be included only with the parameter that defines the input dataset for the tool. If this attribute name is not included, the functional test framework will attempt to determine the data type for the input dataset using the data type sniffers. + This attribute name should be included +only with the parameter that defines the input dataset for the tool. If this +attribute name is not included, the functional test framework will attempt to +determine the data type for the input dataset using the data type sniffers. - dbkey for the referenced input format. + Specifies a ``dbkey`` value for the +referenced input dataset. This is only valid if the corresponding parameter is +of ``type`` ``data``. @@ -739,7 +785,16 @@ Toggle line numbers - Documentation for TestOutput + `` tag sets and generate a +temporary file, which will either be compared with the file named in the +``file`` attribute value or checked against assertions made by a child +``assert_contents`` tag to verify that the tool is functionally correct. + + ]]> @@ -747,37 +802,68 @@ Toggle line numbers - Documentation for name + `` +tag set contained within the tool's ```` tag set. + +]]> - Documentation for file + - Documentation for ftype + - Documentation for sort + This flag causes the lines of the output +to be sorted before they are compared to the expected output. This could be +useful for non-deterministic output. - An alias for file. + An alias for ``file``. - Expected MD5sum for output. + - Checksum expression for output in the form ``hash_type:hash_value`` (e.g. ``sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041``). + @@ -799,7 +885,62 @@ Toggle line numbers - + + + + + + + + + + + + + +``` + +The following demonstrtes a wide variety of XML assertion statements. + +```xml + + + + + + + + + + + + + +``` + +The following demonstrtes verifying XML content with XPath-like expressions. + +```xml + + + + + + +``` + +]]> + + @@ -882,7 +1023,19 @@ Toggle line numbers - Documentation for TestAssertion + + @@ -964,12 +1117,12 @@ Toggle line numbers - ``).]]> + ``).]]> - `` ||).]]> + ``).]]> @@ -2837,9 +2990,12 @@ in the same way that the command line parameters are (discussed in the The following will create a variable called ``$out_file1`` with data type ``pdf``. + +```xml +``` The valid values for format can be found in [/config/datatypes_conf.xml.sample](https://github.com/galaxyproject/galaxy/blob/dev/config/datatypes_conf.xml.sample). @@ -2869,7 +3025,7 @@ conditional blocks should not contain parameters with the same name. - + @@ -2983,14 +3139,28 @@ dataset in the history view. (Default is ``false``.) - Documentation for Collection + `` tag set, and it defines the +output dataset collection description resulting from the tool's execution. The +value of the attribute ``label`` can be acquired from input parameters or +metadata in the same way that the command line parameters are (discussed in the +[``command``](#tool|command) directive). + +Creating collections in tools is covered in-depth in +[planemo's documentation](http://planemo.readthedocs.io/en/latest/writing_advanced.html#creating-collections). + +]]> - - - Documentation for name + + + @@ -3000,28 +3170,40 @@ dataset in the history view. (Default is ``false``.) - Documentation for label + .}``, as +discussed for command line parameters in the ```` tag set section +above. The default label is ``${tool.name} on ${on_string}``. + +]]> - Input collection or dataset to derive output dataset element's format/datatype from. + This is the name of input collection or +dataset to derive output dataset collection's element's format/datatype from. - Input collection to derive collection's type (e.g. ``collection_type``) from. + This is the name of input collection to +derive collection's type (e.g. ``collection_type``) from. - - Input collection or dataset to derive structure of input from (out element count and identifiers). + This is the name of input collection or +dataset to derive "structure" of the output from (output element count and +identifiers). For instance, if the referenced input has three ordered items with +identifiers ``sample1``, ``sample2``, and ``sample3`` - If ``structured_like`` is set, inherit format of outputs from format of corresponding input. + If ``structured_like`` is set, inherit +format of outputs from format of corresponding input. @@ -4381,7 +4563,10 @@ and ``bibtex`` are the only supported options. - Type of comparison to use for test outputs. + Type of comparison to use when comparing +test generated output files to expected output files. Currently valid value are +``diff`` (the default), ``re_match``, ``sim_size``, ``re_match_multiline``, +and ``contains``. From c607727d83c079a2859ed9026d26edbb52f4bcf9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2016 16:15:15 -0400 Subject: [PATCH 094/109] Document the test ``output_collection`` tag. --- galaxy.xsd | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 3397b88de915..16e5229da0c5 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1000,19 +1000,130 @@ The following demonstrtes verifying XML content with XPath-like expressions. - Define test for extra files on corresponding output collection. + + + + + + + + + + + + +``` + +The [CWPair2](https://github.com/galaxyproject/tools-iuc/blob/master/tools/cwpair2/cwpair2.xml) +tool demonstrates that ``element``s can specify a ``compare`` attribute just +like [``output``](#tool|tests|test|output). + +```xml + + + + + + + + + + + + + + + + +``` + +The +[collection_creates_dynamic_nested](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/collection_creates_dynamic_nested.xml) +test tool demonstrates the use of nested ``element`` directives as described +above. Notice also that it tests the output with ``assert_contents`` instead of +supplying a ``file`` attribute. Like hinted at with with ``compare`` attribute +above, the ``element`` tag can specify any of the test attributes that apply to +the [``output``](#tool|tests|test|output) (e.g. ``md5``, ``compare``, ``diff``, +etc...). + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +]]> - - Documentation for name - + `` tag set contained within the tool's ```` tag set. + +]]> - Documentation for type + Expected collection type (e.g. ``list``, ``paired``, +or ``list:paired``). From fc440a87c4883a9244d5e45783e2f27b639319cb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 13 Sep 2016 16:46:31 -0400 Subject: [PATCH 095/109] Fix missing annotation. --- galaxy.xsd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 16e5229da0c5..b7fa49d639ff 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1113,12 +1113,14 @@ etc...). - + `` tag set contained within the tool's ```` tag set. ]]> + @@ -1132,6 +1134,7 @@ or ``list:paired``). + Date: Wed, 14 Sep 2016 08:32:14 -0400 Subject: [PATCH 096/109] Fixes of structure broken with docs. --- galaxy.xsd | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index b7fa49d639ff..d4d75f03f137 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -615,23 +615,6 @@ specified and instead a series of assertions is made about the output. - - - - - Describe assertions about the jobs standard error. - - - - - Describe assertions about the jobs standard error. - - - - - Describe assertions about the jobs standard error. - - @@ -676,7 +659,8 @@ $assertions - Describe assertions about the job's standard error. + Describe assertions about the job's +standard output. $assertions @@ -684,7 +668,8 @@ $assertions - Describe assertions about the jobs standard error. + Describe assertions about the job's +standard error. $assertions @@ -1392,7 +1377,7 @@ between paired inputs of single stranded inputs. - + @@ -3269,7 +3254,7 @@ Creating collections in tools is covered in-depth in - + - - - - - From b74c1a5b0baa6da94ca45f11a1e8e2057fcf2790 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 08:41:40 -0400 Subject: [PATCH 097/109] Use the new action conditional filter. --- galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index d4d75f03f137..4cd7d2c57ec0 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -3608,7 +3608,7 @@ if none of the nested options apply. - + From 195fd46604a64ad8928ead83d6cd6f7f51a2639c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 09:28:01 -0400 Subject: [PATCH 098/109] Add a fictious XML schema with Galaxy stuff. New attributes: - deprecated - best_practices (provides links of the form http://planemo.readthedocs.io/en/latest/standards/docs/best_practices/tool_xml.html#XXX) --- galaxy.xsd | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 4cd7d2c57ec0..e3005f934029 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1,6 +1,7 @@ @@ -10,7 +11,7 @@ - + - + `` tag set described above). @@ -88,7 +89,7 @@ substitution. - + - + Must be unique across all tools; should be lowercase and contain only letters, numbers, and underscores. It allows for tool versioning and metrics of the number of times a tool is used, @@ -153,13 +154,13 @@ among other things. - + This string is what is displayed as a hyperlink in the tool menu. - + This string defaults to ``1.0.0`` if it is not included in the tag. It allows for tool versioning and should be changed with each new version of the tool. @@ -521,7 +522,7 @@ def get_field_components_options( dataset, field_name ): - + tag sets. Any number of tests can be included, @@ -1811,7 +1812,7 @@ displayed on the tool page as the label of the form field - + Short bit of text, rendered on the tool form just below the associated field to provide information about the field. @@ -2018,7 +2019,7 @@ allow access to Python code to generate options for a select list. See - + - + This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. From 4dd8da09b8bd1917094622039adfe01cffa2fd40 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 09:29:56 -0400 Subject: [PATCH 099/109] Remove embedded best practice stuff - the help will have links now. --- galaxy.xsd | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index e3005f934029..bda5cfd6f2ef 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -130,12 +130,6 @@ Query sequence:: ``` -### Best Practices - -From https://galaxy-iuc-standards.readthedocs.org/en/latest/: - -The help tag should be started and finished by a [CDATA tag](https://en.wikipedia.org/wiki/CDATA). - ]]> @@ -2203,12 +2197,6 @@ legacy tools. Older tools may define an ``intepreter`` attribute on the command, but this is deprecated and using the ``$__tool_directory__`` variable is superior. -### Best Practices - -From https://galaxy-iuc-standards.readthedocs.org/en/latest/: - -The ``command`` tag should be started and finished by a [CDATA tag](https://en.wikipedia.org/wiki/CDATA). - ]]> From c4f03079d7fefb2e37790b4d07fe7d0a67990d8d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 14:35:00 -0400 Subject: [PATCH 100/109] Fill out some docs for test sections... --- galaxy.xsd | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 250 insertions(+), 2 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index bda5cfd6f2ef..ccb2ebf39b65 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -673,24 +673,272 @@ $assertions + + +
+ +
+
+ +
+ + + + + + + +``` + +]]> + + + + This value must match the name of the +associated input ``section``. + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +The [tophat2](https://github.com/galaxyproject/tools-devteam/blob/master/tools/tophat2/tophat2_wrapper.xml) +tool demonstrates a real tool that benefits from more structured test cases +using the ``conditional`` test directive. One such test case from that tool is +shown below. + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +]]> + + + This value must match the name of the +associated input ``conditional``. + + + `` elements in the ```` definition. + +```xml + + + + + + + + + + + + + + + +``` + +The second definition in that file demonstrates repeated ```` blocks +allowing multiple instances of a single repeat to be specified. + +```xml + + + + + + + + + + + + + + + + +``` + +]]> + - + + + This value must match the name of the +associated input ``repeat``. + + @@ -710,7 +958,7 @@ some simple examples of parameters. This value must match the name of the -associated input parameter. +associated input parameter (``param``). From 0c6be2095710fbb79bd9b9fdd231e960bb9f2ce8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 08:24:43 -0400 Subject: [PATCH 101/109] Revise automated testing, check against test tools and IUC tools. --- test.sh | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/test.sh b/test.sh index a9bec6e31bb0..874fa3eeafe2 100644 --- a/test.sh +++ b/test.sh @@ -1,18 +1,34 @@ #! /bin/bash -count=`wc -l test.urls | cut -f1 -d' '` -root_url="https://bitbucket.org/galaxy/galaxy-central/raw/c3eefbdaaa1ab242a1c81b65482ef2fbe943a390/" +if [ ! -d galaxy ]; +then + git clone https://github.com/galaxyproject/galaxy.git galaxy +fi +cd galaxy +git pull +cd .. + +if [ ! -d tools-iuc ]; +then + git clone https://github.com/galaxyproject/tools-iuc.git tools-iuc +fi +cd tools-iuc +git pull +cd .. + +count=`wc -l tool_files.list | cut -f1 -d' '` echo "1..$count" count=0 while read p; do count=$((count+1)) - url="$root_url""$p" - result=`curl -ksL "$url" | xmllint --nowarning --noout --schema galaxy.xsd - 2> err.tmp` + path=$p + + result=`planemo normalize --expand_macros "$path" | xmllint --nowarning --noout --schema galaxy.xsd - 2> err.tmp` if [ $? -eq 0 ] then echo "ok $count $url"; else - echo "not ok $count $url"; + echo "not ok $count $path"; cat err.tmp | sed 's/^/ /' fi -done Date: Wed, 14 Sep 2016 15:03:12 -0400 Subject: [PATCH 102/109] Document testing discovered datasets and metadata. --- galaxy.xsd | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index ccb2ebf39b65..445c7a3f2808 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1187,20 +1187,75 @@ The following demonstrtes verifying XML content with XPath-like expressions. + + + + + + + + + +``` + +]]> + - Documentation for name + Name of the metdata element to check. - Documentation for value + Expected value (as a string) of metadata value. - Define test for a discovered dataset on corresponding output. + + + + + + + + + + + + + + + + +``` + +]]> From 8b7ae4bc7b446b0b00758f6cf0c98e7eaece96ce Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 15:07:46 -0400 Subject: [PATCH 103/109] Fix a syntax error. --- galaxy.xsd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/galaxy.xsd b/galaxy.xsd index 445c7a3f2808..6bb3a01fd12d 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -1224,10 +1224,10 @@ provides a demonstration of using this tag. - Date: Wed, 14 Sep 2016 20:58:47 -0400 Subject: [PATCH 104/109] Update the link for multiple outputs. --- galaxy.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy.xsd b/galaxy.xsd index 6bb3a01fd12d..4bccedfd6897 100644 --- a/galaxy.xsd +++ b/galaxy.xsd @@ -3657,7 +3657,8 @@ Galaxy, including: * https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_assign_primary.xml * https://github.com/galaxyproject/galaxy/tree/master/test/functional/tools/multi_output_configured.xml -More information can be found at https://wiki.galaxyproject.org/Admin/Tools/MultipleOutputFiles#Number_of_Output_datasets_cannot_be_determined_until_tool_run. +More information can be found on Planemo's documentation for +[multiple output files](http://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files). ]]> From 3d7ae4e38fea3d7f84368ccb57f107f0c13f9749 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Sep 2016 21:21:26 -0400 Subject: [PATCH 105/109] Relocate Galaxy XSD files into Galaxy directory structure. --- .gitignore | 2 -- .travis.yml | 4 ---- LICENSE => lib/galaxy/tools/xsd/LICENSE | 0 Makefile => lib/galaxy/tools/xsd/Makefile | 0 README.md => lib/galaxy/tools/xsd/README.md | 0 galaxy.jxb => lib/galaxy/tools/xsd/galaxy.jxb | 0 galaxy.xsd => lib/galaxy/tools/xsd/galaxy.xsd | 0 test.sh => lib/galaxy/tools/xsd/test.sh | 0 test.urls => lib/galaxy/tools/xsd/test.urls | 0 9 files changed, 6 deletions(-) delete mode 100644 .gitignore delete mode 100644 .travis.yml rename LICENSE => lib/galaxy/tools/xsd/LICENSE (100%) rename Makefile => lib/galaxy/tools/xsd/Makefile (100%) rename README.md => lib/galaxy/tools/xsd/README.md (100%) rename galaxy.jxb => lib/galaxy/tools/xsd/galaxy.jxb (100%) rename galaxy.xsd => lib/galaxy/tools/xsd/galaxy.xsd (100%) rename test.sh => lib/galaxy/tools/xsd/test.sh (100%) rename test.urls => lib/galaxy/tools/xsd/test.urls (100%) diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 472ea98b6c6a..000000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -generated -*~ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f1462b08227b..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,4 +0,0 @@ -install: - - sudo apt-get install libxml2-utils -script: - - xmllint --noout *.xsd diff --git a/LICENSE b/lib/galaxy/tools/xsd/LICENSE similarity index 100% rename from LICENSE rename to lib/galaxy/tools/xsd/LICENSE diff --git a/Makefile b/lib/galaxy/tools/xsd/Makefile similarity index 100% rename from Makefile rename to lib/galaxy/tools/xsd/Makefile diff --git a/README.md b/lib/galaxy/tools/xsd/README.md similarity index 100% rename from README.md rename to lib/galaxy/tools/xsd/README.md diff --git a/galaxy.jxb b/lib/galaxy/tools/xsd/galaxy.jxb similarity index 100% rename from galaxy.jxb rename to lib/galaxy/tools/xsd/galaxy.jxb diff --git a/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd similarity index 100% rename from galaxy.xsd rename to lib/galaxy/tools/xsd/galaxy.xsd diff --git a/test.sh b/lib/galaxy/tools/xsd/test.sh similarity index 100% rename from test.sh rename to lib/galaxy/tools/xsd/test.sh diff --git a/test.urls b/lib/galaxy/tools/xsd/test.urls similarity index 100% rename from test.urls rename to lib/galaxy/tools/xsd/test.urls From 491c9fd519be37b6c6f57cd4082d981c9f411ba5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Sep 2016 00:08:59 -0400 Subject: [PATCH 106/109] Cleanup XSD for move to Galaxy core. --- lib/galaxy/tools/xsd/Makefile | 15 ---- lib/galaxy/tools/xsd/test.urls | 142 --------------------------------- 2 files changed, 157 deletions(-) delete mode 100644 lib/galaxy/tools/xsd/Makefile delete mode 100644 lib/galaxy/tools/xsd/test.urls diff --git a/lib/galaxy/tools/xsd/Makefile b/lib/galaxy/tools/xsd/Makefile deleted file mode 100644 index f1d2326b5efa..000000000000 --- a/lib/galaxy/tools/xsd/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -SHELL=/bin/bash -.PHONY: all clean tests - -all: tests - -tests: galaxy.xsd galaxy.jxb generated/ObjectFactory.java - sed "s%^%https://bitbucket.org/galaxy/galaxy-central/raw/c3eefbdaaa1ab242a1c81b65482ef2fbe943a390/%" test.urls |\ - while read L; do echo "#$${L}" && curl -ksL "$${L}" | xmllint --noout --schema $< - ; done - - -generated/ObjectFactory.java : galaxy.jxb galaxy.xsd - $(if $(realpath ${JAVA_HOME}/bin/xjc),xjc -b $^) - -clean: - rm -rf generated diff --git a/lib/galaxy/tools/xsd/test.urls b/lib/galaxy/tools/xsd/test.urls deleted file mode 100644 index a7b967442300..000000000000 --- a/lib/galaxy/tools/xsd/test.urls +++ /dev/null @@ -1,142 +0,0 @@ -test/functional/tools/bibtex.xml -test/functional/tools/catDocker.xml -test/functional/tools/simple_constructs.xml -test/functional/tools/collection_paired_test.xml -tools/sr_assembly/velvetg.xml -tools/sr_assembly/velveth.xml -tools/data_source/microbial_import.xml -tools/data_source/wormbase_test.xml -tools/data_source/ucsc_archaea.xml -tools/data_source/worm_modencode.xml -tools/data_source/bed_convert.xml -tools/data_source/gramene_mart.xml -tools/data_source/ebi_sra.xml -tools/data_source/modmine.xml -tools/data_source/import.xml -tools/data_source/echo.xml -tools/data_source/biomart.xml -tools/data_source/ucsc_tablebrowser_test.xml -tools/data_source/flymine_test.xml -tools/data_source/ucsc_proxy.xml -tools/data_source/biomart_test.xml -tools/data_source/access_libraries.xml -tools/data_source/ucsc_tablebrowser.xml -tools/data_source/hbvar.xml -tools/data_source/wormbase.xml -tools/data_source/flymine.xml -tools/data_source/ucsc_tablebrowser_archaea.xml -tools/data_source/metabolicmine.xml -tools/data_source/eupathdb.xml -tools/data_source/yeastmine.xml -tools/data_source/fly_modencode.xml -tools/data_source/upload.xml -tools/data_source/mousemine.xml -tools/data_source/cbi_rice_mart.xml -tools/data_source/hapmapmart.xml -tools/data_source/ucsc_testproxy.xml -tools/data_source/ratmine.xml -tools/data_source/genbank.xml -tools/visualization/LAJ.xml -tools/stats/aggregate_binned_scores_in_intervals.xml -tools/stats/filtering.xml -tools/stats/grouping.xml -tools/stats/gsummary.xml -tools/ngs_simulation/ngs_simulation.xml -tools/maf/maf_by_block_number.xml -tools/maf/maf_to_interval.xml -tools/maf/maf_to_bed.xml -tools/maf/maf_limit_size.xml -tools/maf/vcf_to_maf_customtrack.xml -tools/maf/maf_limit_to_species.xml -tools/maf/maf_filter.xml -tools/maf/maf_split_by_species.xml -tools/maf/maf_to_fasta.xml -tools/maf/genebed_maf_to_fasta.xml -tools/maf/interval2maf_pairwise.xml -tools/maf/interval2maf.xml -tools/maf/maf_stats.xml -tools/maf/maf_thread_for_species.xml -tools/maf/macros.xml -tools/maf/interval_maf_to_merged_fasta.xml -tools/maf/maf_reverse_complement.xml -tools/metag_tools/shrimp_wrapper.xml -tools/metag_tools/blat_wrapper.xml -tools/metag_tools/shrimp_color_wrapper.xml -tools/genomespace/genomespace_exporter.xml -tools/genomespace/genomespace_file_browser_dev.xml -tools/genomespace/genomespace_importer.xml -tools/genomespace/genomespace_file_browser_prod.xml -tools/genomespace/genomespace_file_browser_test.xml -tools/plotting/bar_chart.xml -tools/plotting/boxplot.xml -tools/solid_tools/maq_cs_wrapper.xml -tools/solid_tools/solid_qual_stats.xml -tools/solid_tools/solid_qual_boxplot.xml -tools/sr_mapping/srma_wrapper.xml -tools/sr_mapping/PerM.xml -tools/sr_mapping/bfast_wrapper.xml -tools/sr_mapping/fastq_statistics.xml -tools/sr_mapping/mosaik.xml -tools/extract/extract_genomic_dna.xml -tools/extract/liftOver_wrapper.xml -tools/validation/fix_errors.xml -tools/meme/meme.xml -tools/meme/fimo.xml -tools/next_gen_conversion/solid_to_fastq.xml -tools/next_gen_conversion/fastq_conversions.xml -tools/next_gen_conversion/solid2fastq.xml -tools/next_gen_conversion/fastq_gen_conv.xml -tools/evolution/add_scores.xml -tools/evolution/codingSnps.xml -tools/phenotype_association/master2pg.xml -tools/phenotype_association/pass.xml -tools/phenotype_association/lps.xml -tools/phenotype_association/ldtools.xml -tools/phenotype_association/master2gd_snp.xml -tools/phenotype_association/sift.xml -tools/phenotype_association/linkToDavid.xml -tools/phenotype_association/beam.xml -tools/phenotype_association/linkToGProfile.xml -tools/phenotype_association/gpass.xml -tools/filters/CreateInterval.xml -tools/filters/convert_characters.xml -tools/filters/bed2gff.xml -tools/filters/mergeCols.xml -tools/filters/sorter.xml -tools/filters/trimmer.xml -tools/filters/uniq.xml -tools/filters/wig_to_bigwig.xml -tools/filters/ucsc_gene_bed_to_exon_bed.xml -tools/filters/gff2bed.xml -tools/filters/fileGrep.xml -tools/filters/randomlines.xml -tools/filters/axt_to_fasta.xml -tools/filters/cutWrapper.xml -tools/filters/gtf2bedgraph.xml -tools/filters/axt_to_lav.xml -tools/filters/ucsc_gene_table_to_intervals.xml -tools/filters/pasteWrapper.xml -tools/filters/gff/extract_GFF_Features.xml -tools/filters/gff/gtf_filter_by_attribute_values_list.xml -tools/filters/gff/gff_filter_by_feature_count.xml -tools/filters/gff/gff_filter_by_attribute.xml -tools/filters/grep.xml -tools/filters/fixedValueColumn.xml -tools/filters/catWrapper.xml -tools/filters/condense_characters.xml -tools/filters/headWrapper.xml -tools/filters/wc_gnu.xml -tools/filters/axt_to_concat_fasta.xml -tools/filters/commWrapper.xml -tools/filters/tailWrapper.xml -tools/filters/bed_to_bigbed.xml -tools/filters/lav_to_bed.xml -tools/filters/joiner.xml -tools/filters/changeCase.xml -tools/filters/compare.xml -tools/filters/wiggle_to_simple.xml -tools/filters/secure_hash_message_digest.xml -tools/filters/sff_extractor.xml -tools/filters/remove_beginning.xml -tools/filters/ucsc_gene_bed_to_intron_bed.xml -tools/filters/joiner2.xml From babaf7a2c675648e2851f81d985a72321b4eeb44 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Sep 2016 11:01:00 -0400 Subject: [PATCH 107/109] Address reviewer comments on XSD. - Merge in @bgruening's numerous awesome fixes. - Remove test script with issues that make if not a fit for this repo in that form (noted by @nsoranzo). - Incorporate other changes to the XSD suggested by @nsoranzo. --- lib/galaxy/tools/xsd/galaxy.xsd | 159 ++++++++++++++++++-------------- lib/galaxy/tools/xsd/test.sh | 34 ------- 2 files changed, 92 insertions(+), 101 deletions(-) delete mode 100644 lib/galaxy/tools/xsd/test.sh diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 4bccedfd6897..a6bdcdba6ab8 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -90,7 +90,7 @@ substitution. - seq1 - >ATCG... + ATCG... .. image:: my_figure.png :height: 500 @@ -156,7 +156,7 @@ hyperlink in the tool menu. This string defaults to ``1.0.0`` if it is not -included in the tag. It allows for tool versioning and should be changed with each new version +included in the tag. It allows for tool versioning and should be increased with each new version of the tool. @@ -197,7 +197,7 @@ automatically if profile is set to any version newer than ``16.01``, such as This attribute indicates if -this tool usable within a workflow (defaults to ``true`` for normal tools and +this tool is usable within a workflow (defaults to ``true`` for normal tools and ``false`` for data sources). @@ -246,8 +246,7 @@ complete descriptions of the runtime of a tool. - This value defines the which type of the 3rd party module required by this tool + This value defines the which type of the 3rd party module required by this tool. - Required for package type requirements + For package type requirements this value defines a specific version of the tool dependency. @@ -519,7 +518,7 @@ def get_field_components_options( dataset, field_name ): tag sets. Any number of tests can be included, +Container tag set to specify tests via the tag sets. Any number of tests can be included, and each test is wrapped within separate tag sets. Functional tests are executed via [Planemo](http://planemo.readthedocs.io/) or the [run_tests.sh](https://github.com/galaxyproject/galaxy/blob/dev/run_tests.sh) @@ -547,7 +546,7 @@ the functional test framework. ### Example -The following two tests will tool execute the +The following two tests will execute the [/tools/filters/sorter.xml](https://github.com/galaxyproject/galaxy/blob/dev/tools/filters/sorter.xml) tool. Notice the way that the tool's inputs and outputs are defined. @@ -970,7 +969,7 @@ values that can be assigned to an input parameter. This attribute name should be included -only with the parameter that defines the input dataset for the tool. If this +only with parameters of ``type`` ``data`` for the tool. If this attribute name is not included, the functional test framework will attempt to determine the data type for the input dataset using the data type sniffers. @@ -1054,7 +1053,7 @@ the tool via the functional test framework. @@ -1088,7 +1087,7 @@ and this can be used instead. If specified, the target output's checksum should match the value specified here. This value should have the form ``hash_type:hash_value`` -(e.g. ``sha1$8156d7ca0f46ed7abac98f82e36cfaddb2aca041``). For large static files +(e.g. ``sha1:8156d7ca0f46ed7abac98f82e36cfaddb2aca041``). For large static files it may be inconvenient to upload the entiry file and this can be used instead. ]]> @@ -1526,10 +1525,10 @@ module. - Consists of all tag sets that define the -tool's input parameters. Each <param> tag within the <inputs> tag set + `` tag within the ```` tag set maps to a command line parameter within the [``command``](#tool|command) tag. Most -tools will not need to specify any attributes on this tag itself. +tools will not need to specify any attributes on this tag itself.]]> @@ -1625,7 +1624,7 @@ user selects for this "test" parameter - different UI elements will be shown. These different paths are described by the following the ``when`` blocks shown above. -The following Cheetah block demonstrates the use of the ``conditoinal`` +The following Cheetah block demonstrates the use of the ``conditional`` shown above: ``` @@ -1643,8 +1642,10 @@ conditionally reference ``${input_type.process_obs_metadata}`` with a Cheetah ``if`` statement. A common use of the conditional wrapper is to select between reference data -(managed by the Galaxy deployer or admin - for instance via Data Managers) or -history files for references. A good example tool that demonstrates this is +managed by the Galaxy admins (for instance via +[data managers](https://wiki.galaxyproject.org/Admin/Tools/DataManagers) +) and +history files. A good example tool that demonstrates this is the [Bowtie 2](https://github.com/galaxyproject/tools-devteam/blob/master/tools/bowtie2/bowtie2_wrapper.xml) wrapper. ```xml @@ -1752,7 +1753,7 @@ Cheetah code below. ### Example -This is part is contained in the ```` tag set. +This part is contained in the ```` tag set. ```xml @@ -1768,7 +1769,7 @@ This Cheetah code can be used in the ```` tag set or the ```xml #for $i, $s in enumerate( $series ) rank_of_series=$i - input_path=${s.input} + input_path='${s.input}' x_colom=${s.xcol} y_colom=${s.ycol} #end for @@ -1776,19 +1777,43 @@ This Cheetah code can be used in the ```` tag set or the ### Testing -This is an example test case with multiple repeat elements for the example above. Instances are created by referring to names with a special format: "_|" +This is an example test case with multiple repeat elements for the example above. ```xml - + + + + + + + + + + + + +``` + +See the documentation on the [``repeat`` test directive](#tool|tests|test|repeat). + +An older way to specify repeats in a test is by instances that are created by referring to names with a special format: "_|" + +```xml + + - + + ``` +The test tool [disambiguate_repeats.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/disambiguate_repeats.xml) +demonstrates both testing strategies. + ]]> @@ -1803,22 +1828,22 @@ This is an example test case with multiple repeat elements for the example above - The title of the repeat section, which will be displayed on the tool form + The title of the repeat section, which will be displayed on the tool form. - The minimum number of repeat units + The minimum number of repeat units. - The maximum number of repeat units + The maximum number of repeat units. - The default number of repeat units + The default number of repeat units. @@ -1844,9 +1869,9 @@ The XML configuration is relatively trivial for sections: ```xml -
- -
+
+ +
``` @@ -1865,17 +1890,17 @@ Further examples can be found in the [test case](https://github.com/galaxyprojec - The internal key used for the section + The internal key used for the section. - Human readable label for the section + Human readable label for the section. - Whether the section should be expanded by default or not. If not, the default set values are used + Whether the section should be expanded by default or not. If not, the default set values are used. @@ -2007,7 +2032,7 @@ $attribute_list:hierarchy $attribute_list:format,collection_type -The following will create a parameter that only accepts paired FASTQ files. +The following will create a parameter that only accepts paired FASTQ files grouped into a collection. ##### Examples @@ -2186,7 +2211,7 @@ Set to ``false`` to not force user to select an option in the list. - >Used only if the ``type`` attribute + Used only if the ``type`` attribute value is ``data_column``, if ``true`` Galaxy assumes first row of ``data_ref`` is a header and builds the select list with these values rather than the more generic ``c1`` ... ``cN``. @@ -2220,7 +2245,7 @@ value is ``drill_down``, this attribute determines the drill down is ``recursive`` or ``exact``. - + Set to ``true`` if the ``boolean`` parameter should be checked (or ``true``) by default. @@ -2352,29 +2377,29 @@ and uses whitespace to make a complex command very readable can be found in tool. ```xml - + "${output}" - +]]]]> ``` The following example (taken from [xpath](https://github.com/galaxyproject/tools-iuc/blob/master/tools/xpath/xpath.xml) tool) @@ -2402,13 +2427,13 @@ code file wraps a metadata collection to return metadata parameters wrapped according to the Metadata spec. ```xml - + + --output '$output' +]]]]> ``` In additon to demonstrating accessing metadata, this example demonstrates: @@ -2450,7 +2475,7 @@ even though they don't appear in your tool's parameters. Name | Description ---- | ----------- -``$__tool_directory__`` | The directory the tool currently resides in (new in 15.03) +``$__tool_directory__`` | The directory the tool description (XML file) currently resides in (new in 15.03) ``$__new_file_path__`` | ``config/galaxy.ini``'s ``new_file_path`` value ``$__tool_data_path__`` | ``config/galaxy.ini``'s tool_data_path value ``$__root_dir__`` | Top-level Galaxy source directory made absolute via ``os.path.abspath()`` @@ -2460,7 +2485,7 @@ Name | Description ``$__app__`` | The ``galaxy.app.UniverseApplication`` instance, gives access to all other configuration file variables (e.g. $__app__.config.output_size_limit). Should be used as a last resort, may go away in future releases. Additional runtime properties are available as environment variables. Since these -are not Cheetah variables (the values aren't available until runtme) these should likely +are not Cheetah variables (the values aren't available until runtime) these should likely be escaped with a backslash (``\``) when appearing in ``command`` or ``configfile`` elements. Name | Description @@ -3008,9 +3033,9 @@ ASCII digits, and ``_``. ``` - - - + + + ``` @@ -3019,12 +3044,12 @@ be dropped from the input. ``` - - - - - - + + + + + + ``` ]]> @@ -3080,7 +3105,7 @@ valid input for the mapping to occur. Preset lists include default and none as w - Add a character to add to the list of valid characters + Add a character to the list of valid characters. @@ -3098,7 +3123,7 @@ Preset lists include default and none as well as those available from string.* ( - A character to remove from the list of valid characters + A character to remove from the list of valid characters. @@ -3232,10 +3257,10 @@ While this fragment from maf_to_interval.xml demonstrates removing items. display="checkboxes" multiple="true" help="The species matching the dbkey of the alignment is always included. A separate history item will be created for each species."> - - - - + + + + ``` diff --git a/lib/galaxy/tools/xsd/test.sh b/lib/galaxy/tools/xsd/test.sh deleted file mode 100644 index 874fa3eeafe2..000000000000 --- a/lib/galaxy/tools/xsd/test.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash -if [ ! -d galaxy ]; -then - git clone https://github.com/galaxyproject/galaxy.git galaxy -fi -cd galaxy -git pull -cd .. - -if [ ! -d tools-iuc ]; -then - git clone https://github.com/galaxyproject/tools-iuc.git tools-iuc -fi -cd tools-iuc -git pull -cd .. - -count=`wc -l tool_files.list | cut -f1 -d' '` -echo "1..$count" -count=0 -while read p; do - count=$((count+1)) - path=$p - - result=`planemo normalize --expand_macros "$path" | xmllint --nowarning --noout --schema galaxy.xsd - 2> err.tmp` - if [ $? -eq 0 ] - then - echo "ok $count $url"; - else - echo "not ok $count $path"; - cat err.tmp | sed 's/^/ /' - fi -done Date: Thu, 15 Sep 2016 14:50:39 -0400 Subject: [PATCH 108/109] Update the LICENSE for the XSD file. --- lib/galaxy/tools/xsd/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/xsd/LICENSE b/lib/galaxy/tools/xsd/LICENSE index bfa63c52c28e..09eb8b9bca28 100644 --- a/lib/galaxy/tools/xsd/LICENSE +++ b/lib/galaxy/tools/xsd/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2011-2015 John Chilton, Jean-Frédéric Berthelot, Pierre Lindenbaum +Copyright (c) 2011-2016 John Chilton, Jean-Frédéric Berthelot, Pierre Lindenbaum Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 9d24d55cb5e515b8e8f9654e070ae17aae864367 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Sep 2016 15:09:18 -0400 Subject: [PATCH 109/109] Add XSD contributors to CONTRIBUTORS.md. --- CONTRIBUTORS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d97fedfb49be..4ef53e18e99f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -48,6 +48,7 @@ The following individuals have contributed code to Galaxy: * Eric Enns * fescudie * Dorine Francheteau +* Jean-Frédéric (@JeanFred on Github) * Jaime Frey * Carrie Ganote * Ryan Golhar @@ -77,6 +78,7 @@ The following individuals have contributed code to Galaxy: * Simone Leo * Kanwei Li * Michael Li +* Pierre Lindenbaum * Mikael Loaec * Philip Mabon * Remi Marenco