gr[m] = []
gr[m].append(match.group(0))
+class EvaluationError(Exception):
+ pass
+
# Return the name of variable ('var') that will take on each value in 'items'
# when performing an inner substitution
def var_items(p, c, key):
+ if key not in c:
+ raise EvaluationError("'%s' was expected in 'p' but is missing" % key)
+
if "var" in c:
+ if not isinstance(c["var"], basestring):
+ raise EvaluationError("Value of 'var' must be a string")
# Var specifies the variable name for inner parameter substitution
return (c["var"], get_items(p, c[key]))
else:
# The component function ('key') value is a list, so return the list
- # directly with no parameter substition.
+ # directly with no parameter selected.
if isinstance(c[key], list):
return (None, get_items(p, c[key]))
-
- # check if c[key] is a string that looks like a parameter
- m = re.match("^\$\((.*)\)$", c[key])
- if m and m.group(1) in p:
- return (m.group(1), get_items(p, c[key]))
+ elif isinstance(c[key], basestring):
+ # check if c[key] is a string that looks like a parameter
+ m = re.match("^\$\((.*)\)$", c[key])
+ if m and m.group(1) in p:
+ return (m.group(1), get_items(p, c[key]))
+ else:
+ # backwards compatible, foreach specifies bare parameter name to use
+ return (c[key], get_items(p, p[c[key]]))
else:
- # backwards compatible, foreach specifies bare parameter name to use
- return (c[key], get_items(p, p[c[key]]))
+ raise EvaluationError("Value of '%s' must be a string or list" % key)
# "p" is the parameter scope, "c" is the item to be expanded.
# If "c" is a dict, apply function expansion.
if isinstance(c, dict):
if "foreach" in c and "command" in c:
var, items = var_items(p, c, "foreach")
+ if var is None:
+ raise EvaluationError("Must specify 'var' in foreach")
r = []
for i in items:
params = copy.copy(p)
params[var] = i
r.append(expand_item(params, c["command"]))
return r
- if "list" in c and "index" in c and "command" in c:
+ elif "list" in c and "index" in c and "command" in c:
var, items = var_items(p, c, "list")
+ if var is None:
+ raise EvaluationError("Must specify 'var' in list")
params = copy.copy(p)
params[var] = items[int(c["index"])]
return expand_item(params, c["command"])
- if "regex" in c:
+ elif "regex" in c:
pattern = re.compile(c["regex"])
if "filter" in c:
- var, items = var_items(p, c, "filter")
+ _, items = var_items(p, c, "filter")
return [i for i in items if pattern.match(i)]
elif "group" in c:
- var, items = var_items(p, c, "group")
+ _, items = var_items(p, c, "group")
groups = {}
for i in items:
match = pattern.match(i)
add_to_group(groups, match)
return [groups[k] for k in groups]
elif "extract" in c:
- var, items = var_items(p, c, "extract")
+ _, items = var_items(p, c, "extract")
r = []
for i in items:
match = pattern.match(i)
if match:
r.append(list(match.groups()))
return r
- if "batch" in c and "size" in c:
- var, items = var_items(p, c, "batch")
+ elif "batch" in c and "size" in c:
+ _, items = var_items(p, c, "batch")
sz = int(c["size"])
r = []
for j in xrange(0, len(items), sz):
r.append(items[j:j+sz])
return r
+ raise EvaluationError("Missing valid list context function")
elif isinstance(c, list):
return [expand_item(p, arg) for arg in c]
elif isinstance(c, basestring):
else:
return subst.do_substitution(p, c)
- raise Exception("expand_item() unexpected parameter type %s" % (type(c))
+ raise EvaluationError("expand_item() unexpected parameter type %s" % (type(c))
# Evaluate in a list context
# "p" is the parameter scope, "value" will be evaluated
with open(value) as f:
items = [line.rstrip("\r\n") for line in f]
return items
- raise Exception("get_items did not yield a list")
+ raise EvaluationError("get_items did not yield a list")
stdoutname = None
stdoutfile = None
<pre>
{
- "command": ["echo", "$(file $(a))"],
- "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2"
+ "a": "c1bad4b39ca5a924e481008009d94e32+210/var-GS000016015-ASM.tsv.bz2",
+ "command": ["echo", "$(file $(a))"]
}
</pre>
h2. List context
-When a parameter is evaluated in a list context, that means its value should evaluate to a list instead of a string. Parameter values can be a static list (as demonstrated above), a path to a file, a path to a directory, or a JSON object describing a list context function.
+Where specified by the documentation, parameters may be evaluated in a "list context". That means the value will evaluate to a list instead of a string. Parameter values can be a static list, a path to a file, a path to a directory, or a JSON object describing a list context function.
-If the value is a static list, it will evaluate the list items for parameter substation and list functions.
+If the value is a string, it is interpreted as a path. If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped). If the path specifies a directory, produce a list containing all of the entries in the directory. Note that parameter expansion is not performed on list items produced this way.
-If the value is a string, it is interpreted as a path. If the path specifies a regular file, that file will be opened as a text file and produce a list with one item for each line in the file (end-of-line characters will be stripped). If the path specifies a directory, produce a list containing all of the entries in the directory. Note that parameter expansion is not performed on lists produced this way.
+If the value is a static list, it will evaluate each item and return the expanded list. Each item may be a string (evaluated for parameter substitution), a list (recursively evaluated), or a JSON object (indicating a list function, described below).
If the value is a JSON object, it is evaluated as a list function described below.
h2. List functions
-When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions. In the following functions, you can either specify the name of a user parameter to act on or provide list value directly in line, for example, the following two fragments yield the same result:
+When @run-command@ is evaluating a list (such as "command"), in addition to string parameter substitution, you can use list item functions. In the following functions, you can either specify the name of a user parameter to act on (@"$(a)"@ in the first example) or provide list value directly in line, for example, the following two fragments yield the same result:
<pre>
{
- "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"foreach": "$(a)",
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", {"foreach": ["alice", "bob"], "var":"a", "command": ["--something", "$(a)"]}],
+ "command": ["echo", {"foreach": ["alice", "bob"],
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
Note: when you provide the list inline with "foreach" or "index", you must include the "var" parameter to specify the substitution variable name to use when evaluating the command fragment.
-You can also nest functions:
+You can also nest functions. This filters @["alice", "bob"]@ on the regular expression @"b.*"@ to get the list @["bob"]@, assigns @a_var@ to each value of the list, then expands @"command"@ to get @["--something", "bob"]@.
<pre>
{
- "command": ["echo", {"foreach": {"filter": ["alice", "bob"], "regex": "b.*"},
- "var":"a",
- "command": ["--something", "$(a)"]}]
+ "command": ["echo", {"foreach": {"filter": ["alice", "bob"],
+ "regex": "b.*"},
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", {"foreach": "$(a)", "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"foreach": "$(a)",
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", {"list": "$(a)", "var":"a", "index": 1, "command": ["--something", "$(a)"]}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"list": "$(a)",
+ "var": "a_var",
+ "index": 1,
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", {"filter": "$(a)", "regex": "b.*"}],
- "a": ["alice", "bob"]
+ "a": ["alice", "bob"],
+ "command": ["echo", {"filter": "$(a)",
+ "regex": "b.*"}]
}
</pre>
h3. group
-Generate a list of lists, where items are grouped on common subexpression match. Items which don't match the regular expression are excluded. The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
+Generate a list of lists, where items are grouped on common subexpression match. Items which don't match the regular expression are excluded. In the following example, the subexpression is @(a?)@, resulting in two groups, strings that contain the letter 'a' and strings that do not. The following example evaluates to @["echo", "--group", "alice", "carol", "dave", "--group", "bob"]@:
<pre>
{
- "command": ["echo", {"foreach": "$(b)", "command":["--group", {"foreach": "b", "command":"$(b)"}]}],
"a": ["alice", "bob", "carol", "dave"],
- "b": {"group": "a", "regex": "[^a]*(a?).*"}
+ "b": {"group": "$(a)",
+ "regex": "[^a]*(a?).*"},
+ "command": ["echo", {"foreach": "$(b)",
+ "var": "b_var",
+ "command": ["--group",
+ {"foreach": "$(b_var)",
+ "var": "c_var",
+ "command": "$(c_var)"}]}]
}
</pre>
<pre>
{
- "command": ["echo", {"foreach": "$(b)", "command":[{"foreach": "$(b)", "command":"$(b)"}]}],
"a": ["alice", "bob", "carol", "dave"],
- "b": {"extract": "a", "regex": "(.+)(a)(.*)"}
+ "b": {"extract": "$(a)",
+ "regex": "(.+)(a)(.*)"},
+ "command": ["echo", {"foreach": "$(b)",
+ "var": "b_var",
+ "command": [{"foreach": "$(b_var)",
+ "var": "c_var",
+ "command": "$(c_var)"}]}]
}
</pre>
<pre>
{
- "command": ["echo", {"foreach":{"batch": "$(a)", "size": 2}, "var":"a", "command":["--something", "$(a)"]}],
- "a": ["alice", "bob", "carol", "dave"]
+ "a": ["alice", "bob", "carol", "dave"],
+ "command": ["echo", {"foreach":{"batch": "$(a)",
+ "size": 2},
+ "var": "a_var",
+ "command": ["--something", "$(a_var)"]}]
}
</pre>
<pre>
{
- "command": ["echo", "$(a)", "$(b)"],
- "task.foreach": ["a", "b"],
"a": ["alice", "bob"],
- "b": ["carol", "dave"]
+ "b": ["carol", "dave"],
+ "task.foreach": ["a", "b"],
+ "command": ["echo", "$(a)", "$(b)"]
}
</pre>