pj
end
- def pipieline_log_history(job_uuids)
+ def pipeline_log_history(job_uuids)
results = []
log_history = Log.where(event_type: 'stderr',
<% end %>
<% end %>
-<% pipeline_jobs = [] %>
<% pipeline_job_uuids = [] %>
<% if !@object.state.in? ['New', 'Ready', 'Paused'] %>
<colgroup>
<col style="width: 15%" />
<col style="width: 20%" />
- <col style="width: 15%" />
<col style="width: 12%" />
<col style="width: 12%" />
- <col style="width: 26%" />
+ <col style="width: 45%" />
</colgroup>
<thead>
<tr>
</th><th>
script, version
</th><th>
- Job
- </th><th>
- progress
+ job
<%# format:'js' here helps browsers avoid using the cached js
content in html context (e.g., duplicate tab -> see
javascript) %>
<tbody>
<% render_pipeline_jobs.each do |pj| %>
<% if pj[:job].andand[:uuid]
- pipeline_jobs << pj[:job]
pipeline_job_uuids << pj[:job][:uuid]
end %>
<tr>
</td><td>
<%= pj[:script] %>
<br /><span class="deemphasize"><%= pj[:script_version] %></span>
- </td><td>
- <% if pj[:job].andand[:uuid] %>
- <%= link_to("..."+pj[:job][:uuid].last(15), job_url(id: pj[:job][:uuid])) %>
- <% end %>
</td><td>
<%= pj[:progress_bar] %>
+ <% if @object.state == 'Complete' || @object.state == 'Failed' %>
+ <% if pj[:job].andand[:uuid] %>
+ <span class="deemphasize">
+ <%= link_to("..."+pj[:job][:uuid].last(15), job_url(id: pj[:job][:uuid])) %>
+ </span>
+
+ <% current_job = Job.find(pj[:job][:uuid]) rescue nil %>
+ <% if current_job.andand[:log] %>
+ <% fixup = /([a-f0-9]{32}\+\d+)(\+?.*)/.match(current_job[:log])%>
+ <% Collection.limit(1).where(uuid: fixup[1]).each do |c| %>
+ <% c.files.each do |file| %>
+ <br/><span class="deemphasize">
+ <a href="<%= collection_path(current_job[:log]) %>/<%= file[1] %>?disposition=inline&size=<%= file[2] %>">log</a>
+ </span>
+ <% end %>
+ <% end %>
+ <% end %>
+ <% end %>
+ <% end %>
</td><td>
<%= render(partial: 'job_status_label',
locals: { :j => pj[:job] }) %>
<% end %>
<% if !pipeline_job_uuids.empty? %>
- <h4>Log for pipeline</h4>
- <% log_history = pipieline_log_history(pipeline_job_uuids) %>
+ <h4>Log messages from running jobs</h4>
+ <% log_history = pipeline_log_history(pipeline_job_uuids) %>
<div id="pipeline_event_log_history_div">
<% log_history.each do |entry| %>
<%=entry%><br/>
</div>
<div class="arv-log-event-listener arv-log-event-handler-append-logs" id="pipeline_event_log_div" data-object-uuids="<%=pipeline_job_uuids.join(" ")%>"/>
<% end %>
- <% else %> <%# Not running. Must be done. %>
- <h4>Log for pipeline</h4>
- <div id="pipeline_event_log_history_div">
- <% pipeline_jobs.each do |j| %>
- <% if j[:log] %>
- <% fixup = /([a-f0-9]{32}\+\d+)(\+?.*)/.match(j[:log])%>
- <% Collection.limit(1).where(uuid: fixup[1]).each do |c| %>
- <% c.files.each do |file| %>
- <a href="<%= collection_path(j[:log]) %>/<%= file[1] %>?disposition=inline&size=<%= file[2] %>"><%=j[:uuid]%> Log</a><br/>
- <% end %>
- <% end %>
- <% end %>
- <% end %>
- </div>
<% end %>
<% else %> <%# State new or ready or paused %>
Next, on the Arvados virtual machine, clone your Git repository:
<notextile>
-<pre><code>~$ <span class="userinput">git clone git@git.{{ site.arvados_api_host }}:<b>you</b>.git</span>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone git@git.{{ site.arvados_api_host }}:<b>you</b>.git</span>
Cloning into '<b>you</b>'...</code></pre>
</notextile>
-This will create a Git repository in the directory called *@you@*.
+This will create a Git repository in the directory called *@you@* in your home directory. Say yes when prompted to continue with connection.
+Ignore any warning that you are cloning an empty repository.
{% include 'notebox_begin' %}
For more information about using Git, try
<notextile>
<pre><code>~/<b>you</b>/crunch_scripts$ <span class="userinput">git commit -m"my first script"</span>
[master (root-commit) 27fd88b] my first script
- 1 file changed, 33 insertions(+)
+ 1 file changed, 45 insertions(+)
create mode 100755 crunch_scripts/hash.py</code></pre>
</notextile>
Compressing objects: 100% (2/2), done.
Writing objects: 100% (4/4), 682 bytes, done.
Total 4 (delta 0), reused 0 (delta 0)
-To git@git.qr1hi.arvadosapi.com:you.git
+To git@git.qr1hi.arvadosapi.com:<b>you</b>.git
* [new branch] master -> master</code></pre>
</notextile>
* @return Set
* @throws Exception
*/
- public Set<String> getAvailableParametersForMethod(String resourceName, String methodName)
+ public Map<String,List<String>> getAvailableParametersForMethod(String resourceName, String methodName)
throws Exception {
RestMethod method = getMatchingMethod(resourceName, methodName);
- Set<String> parameters = method.getParameters().keySet();
- Request request = method.getRequest();
- if (request != null) {
- Object requestProperties = request.get("properties");
- if (requestProperties != null) {
- if (requestProperties instanceof Map) {
- Map properties = (Map)requestProperties;
- Set<String> propertyKeys = properties.keySet();
- if (propertyKeys.size()>0) {
- try {
- propertyKeys.addAll(parameters);
- return propertyKeys;
- } catch (Exception e){
- logger.error(e);
+ Map<String, List<String>> parameters = new HashMap<String, List<String>>();
+ List<String> requiredParameters = new ArrayList<String>();
+ List<String> optionalParameters = new ArrayList<String>();
+ parameters.put ("required", requiredParameters);
+ parameters.put("optional", optionalParameters);
+
+ try {
+ // get any request parameters
+ Request request = method.getRequest();
+ if (request != null) {
+ Object required = request.get("required");
+ Object requestProperties = request.get("properties");
+ if (requestProperties != null) {
+ if (requestProperties instanceof Map) {
+ Map properties = (Map)requestProperties;
+ Set<String> propertyKeys = properties.keySet();
+ for (String property : propertyKeys) {
+ if (Boolean.TRUE.equals(required)) {
+ requiredParameters.add(property);
+ } else {
+ optionalParameters.add(property);
+ }
}
}
}
}
+
+ // get other listed parameters
+ Map<String,JsonSchema> methodParameters = method.getParameters();
+ for (Map.Entry<String, JsonSchema> entry : methodParameters.entrySet()) {
+ if (Boolean.TRUE.equals(entry.getValue().getRequired())) {
+ requiredParameters.add(entry.getKey());
+ } else {
+ optionalParameters.add(entry.getKey());
+ }
+ }
+ } catch (Exception e){
+ logger.error(e);
}
+
return parameters;
}
}
}
+ /**
+ * Convert the input parameter into its equivalent json string.
+ * Add this json string value to the parameters map to be sent to server.
+ * @param argName
+ * @param parameters
+ * @param parameterName
+ * @param parameter
+ * @param parameterValue
+ * @throws Exception
+ */
private void putParameter(String argName, Map<String, Object> parameters,
String parameterName, JsonSchema parameter, Object parameterValue)
throws Exception {
value = new BigInteger(parameterValue.toString());
} else if ("float".equals(parameter.getType())) {
value = new BigDecimal(parameterValue.toString());
+ } else if ("Java.util.Calendar".equals(parameter.getType())) {
+ value = new BigDecimal(parameterValue.toString());
} else if (("array".equals(parameter.getType())) ||
("Array".equals(parameter.getType()))) {
if (parameterValue.getClass().isArray()){
parameters.put(parameterName, value);
}
+ /**
+ * Convert the given input array into json string before sending to server.
+ * @param parameterValue
+ * @return
+ */
private String getJsonValueFromArrayType (Object parameterValue) {
String arrayStr = Arrays.deepToString((Object[])parameterValue);
- arrayStr = arrayStr.substring(1, arrayStr.length()-1);
+
+ // we can expect either an array of array objects or an array of objects
+ if (arrayStr.startsWith("[[") && arrayStr.endsWith("]]")) {
+ Object[][] array = new Object[1][];
+ arrayStr = arrayStr.substring(2, arrayStr.length()-2);
+ String jsonStr = getJsonStringForArrayStr(arrayStr);
+ String value = "[" + jsonStr + "]";
+ return value;
+ } else {
+ arrayStr = arrayStr.substring(1, arrayStr.length()-1);
+ return (getJsonStringForArrayStr(arrayStr));
+ }
+ }
+
+ private String getJsonStringForArrayStr(String arrayStr) {
Object[] array = arrayStr.split(",");
Object[] trimmedArray = new Object[array.length];
for (int i=0; i<array.length; i++){
trimmedArray[i] = array[i].toString().trim();
}
- String jsonString = JSONArray.toJSONString(Arrays.asList(trimmedArray));
- String value = "["+ jsonString +"]";
-
+ String value = JSONArray.toJSONString(Arrays.asList(trimmedArray));
return value;
}
+ /**
+ * Convert the given input List into json string before sending to server.
+ * @param parameterValue
+ * @return
+ */
private String getJsonValueFromListType (Object parameterValue) {
List paramList = (List)parameterValue;
Object[] array = new Object[paramList.size()];
- String arrayStr = Arrays.deepToString(paramList.toArray(array));
- arrayStr = arrayStr.substring(1, arrayStr.length()-1);
- array = arrayStr.split(",");
- Object[] trimmedArray = new Object[array.length];
- for (int i=0; i<array.length; i++){
- trimmedArray[i] = array[i].toString().trim();
- }
- String jsonString = JSONArray.toJSONString(Arrays.asList(trimmedArray));
- String value = "["+ jsonString +"]";
-
- return value;
+ Arrays.deepToString(paramList.toArray(array));
+ return (getJsonValueFromArrayType(array));
}
+ /**
+ * Convert the given input map into json string before sending to server.
+ * @param parameterValue
+ * @return
+ */
private String getJsonValueFromMapType (Object parameterValue) {
JSONObject json = new JSONObject((Map)parameterValue);
return json.toString();
import java.io.File;
import java.io.FileInputStream;
+import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Map response = arv.call("links", "list", params);
assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
- String[] filters = new String[3];
- filters[0] = "name";
- filters[1] = "=";
- filters[2] = "can_manage";
-
+ String[][] filters = new String[1][];
+ String[] condition = new String[3];
+ condition[0] = "name";
+ condition[1] = "=";
+ condition[2] = "can_manage";
+ filters[0] = condition;
params.put("filters", filters);
response = arv.call("links", "list", params);
Map response = arv.call("links", "list", params);
assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
- List<String> filters = new ArrayList<String>();
- filters.add("name");
- filters.add("is_a");
- filters.add("can_manage");
-
+ List<List> filters = new ArrayList<List>();
+ List<String> condition = new ArrayList<String>();
+ condition.add("name");
+ condition.add("is_a");
+ condition.add("can_manage");
+ filters.add(condition);
params.put("filters", filters);
response = arv.call("links", "list", params);
assertFalse("Expected no can_manage in response", response.toString().contains("\"name\":\"can_manage\""));
}
+ @Test
+ public void testGetLinksWithTimestampFilters() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+
+ Map<String, Object> params = new HashMap<String, Object>();
+
+ Map response = arv.call("links", "list", params);
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+
+ // get links created "tomorrow". Expect none in response
+ Calendar calendar = new GregorianCalendar();
+ calendar.setTime(new Date());
+ calendar.add(Calendar.DAY_OF_MONTH, 1);
+
+ Object[][] filters = new Object[1][];
+ Object[] condition = new Object[3];
+ condition[0] = "created_at";
+ condition[1] = ">";
+ condition[2] = calendar.get(Calendar.YEAR) + "-" + (calendar.get(Calendar.MONTH)+1) + "-" + calendar.get(Calendar.DAY_OF_MONTH);
+ filters[0] = condition;
+ params.put("filters", filters);
+
+ response = arv.call("links", "list", params);
+
+ assertEquals("Expected links.list in response", "arvados#linkList", response.get("kind"));
+ int items_avail = ((BigDecimal)response.get("items_available")).intValue();
+ assertEquals("Expected zero links", items_avail, 0);
+ }
+
@Test
public void testGetLinksWithWhereClause() throws Exception {
Arvados arv = new Arvados("arvados", "v1");
@Test
public void testGetAvailableParametersForUsersGetMethod() throws Exception {
Arvados arv = new Arvados("arvados", "v1");
- Set<String> parameters = arv.getAvailableParametersForMethod("users", "get");
+ Map<String,List<String>> parameters = arv.getAvailableParametersForMethod("users", "get");
assertNotNull("Expected parameters", parameters);
- assertTrue("Excected uuid parameter for get method for users", parameters.contains("uuid"));
+ assertTrue("Excected uuid parameter for get method for users", parameters.get("required").contains("uuid"));
}
@Test
public void testGetAvailableParametersForUsersCreateMethod() throws Exception {
Arvados arv = new Arvados("arvados", "v1");
- Set<String> parameters = arv.getAvailableParametersForMethod("users", "create");
+ Map<String,List<String>> parameters = arv.getAvailableParametersForMethod("users", "create");
+ assertNotNull("Expected parameters", parameters);
+ assertTrue("Excected user parameter for get method for users", parameters.get("required").contains("user"));
+ }
+
+ @Test
+ public void testGetAvailableParametersForUsersListMethod() throws Exception {
+ Arvados arv = new Arvados("arvados", "v1");
+ Map<String,List<String>> parameters = arv.getAvailableParametersForMethod("users", "list");
assertNotNull("Expected parameters", parameters);
- assertTrue("Excected user parameter for create method for users", parameters.contains("user"));
+ assertTrue("Excected no required parameter for list method for users", parameters.get("required").size() == 0);
+ assertTrue("Excected some optional parameters for list method for users", parameters.get("optional").contains("filters"));
}
}
\ No newline at end of file
started: false,
sent_int: 0,
job_auth: job_auth,
+ stderr_buf_to_flush: '',
stderr_flushed_at: 0
}
i.close
end
if stderr_buf
- if stderr_buf.index "\n"
- lines = stderr_buf.lines("\n").to_a
+ j[:stderr_buf] << stderr_buf
+ if j[:stderr_buf].index "\n"
+ lines = j[:stderr_buf].lines("\n").to_a
+ if j[:stderr_buf][-1] == "\n"
+ j[:stderr_buf] = ''
+ else
+ j[:stderr_buf] = lines.pop
+ end
lines.each do |line|
$stderr.print "#{job_uuid} ! " unless line.index(job_uuid)
$stderr.puts line
- log_msg = "#{Time.now.ctime.to_s} #{line.strip}"
- j[:stderr_buf] << (log_msg + " \n")
+ pub_msg = "#{Time.now.ctime.to_s} #{line.strip} \n"
+ j[:stderr_buf_to_flush] << pub_msg
end
- if (LOG_BUFFER_SIZE < j[:stderr_buf].size) || ((j[:stderr_flushed_at]+1) < Time.now.to_i)
+ if (LOG_BUFFER_SIZE < j[:stderr_buf_to_flush].size) || ((j[:stderr_flushed_at]+1) < Time.now.to_i)
write_log j
- j[:stderr_flushed_at] = Time.now.to_i
end
end
end
# send message to log table. we want these records to be transient
def write_log running_job
- if (running_job && running_job[:stderr_buf] != '')
+ begin
+ if (running_job && running_job[:stderr_buf_to_flush] != '')
log = Log.new(object_uuid: running_job[:job].uuid,
event_type: 'stderr',
owner_uuid: running_job[:job].owner_uuid,
- properties: {"text" => running_job[:stderr_buf]})
+ properties: {"text" => running_job[:stderr_buf_to_flush]})
log.save!
- running_job[:stderr_buf] = ''
+ running_job[:stderr_buf_to_flush] = ''
running_job[:stderr_flushed_at] = Time.now.to_i
end
rescue
running_job[:stderr_buf] = "Failed to write logs \n"
+ running_job[:stderr_buf_to_flush] = ''
running_job[:stderr_flushed_at] = Time.now.to_i
end
end
projects / arvados.git / commitdiff