# Dev/test SSL certificates
/self-signed.key
/self-signed.pem
+
+# Generated git-commit.version file
+/git-commit.version
# exception here than in a template.)
unless current_user.nil?
begin
+ my_starred_projects current_user
build_my_wanted_projects_tree current_user
rescue ArvadosApiClient::ApiError
# Fall back to the default-setting code later.
end
@starred_projects ||= []
@my_wanted_projects_tree ||= []
- @my_project_tree ||= []
- @shared_project_tree ||= []
render_error(err_opts)
end
links = Link.filter([['tail_uuid', '=', user.uuid],
['link_class', '=', 'star'],
['head_uuid', 'is_a', 'arvados#group']]).select(%w(head_uuid))
- uuids =links.collect { |x| x.head_uuid }
+ uuids = links.collect { |x| x.head_uuid }
starred_projects = Group.filter([['uuid', 'in', uuids]]).order('name')
@starred_projects = starred_projects.results
end
sorted_paths.call buildtree.call(children_of, 'me')
end
- helper_method :my_project_tree
- def my_project_tree
- build_project_trees
- @my_project_tree
- end
-
- helper_method :shared_project_tree
- def shared_project_tree
- build_project_trees
- @shared_project_tree
- end
-
- def build_project_trees
- return if @my_project_tree and @shared_project_tree
- parent_of = {current_user.uuid => 'me'}
- all_projects.each do |ob|
- parent_of[ob.uuid] = ob.owner_uuid
- end
- children_of = {false => [], 'me' => [current_user]}
- all_projects.each do |ob|
- if ob.owner_uuid != current_user.uuid and
- not parent_of.has_key? ob.owner_uuid
- parent_of[ob.uuid] = false
- end
- children_of[parent_of[ob.uuid]] ||= []
- children_of[parent_of[ob.uuid]] << ob
- end
- buildtree = lambda do |children_of, root_uuid=false|
- tree = {}
- children_of[root_uuid].andand.each do |ob|
- tree[ob] = buildtree.call(children_of, ob.uuid)
- end
- tree
- end
- sorted_paths = lambda do |tree, depth=0|
- paths = []
- tree.keys.sort_by { |ob|
- ob.is_a?(String) ? ob : ob.friendly_link_name
- }.each do |ob|
- paths << {object: ob, depth: depth}
- paths += sorted_paths.call tree[ob], depth+1
- end
- paths
- end
- @my_project_tree =
- sorted_paths.call buildtree.call(children_of, 'me')
- @shared_project_tree =
- sorted_paths.call({'Projects shared with me' =>
- buildtree.call(children_of, false)})
- end
-
helper_method :get_object
def get_object uuid
if @get_object.nil? and @objects
<div class="modal-body">
<div class="selectable-container" style="height: 15em; overflow-y: scroll">
- <% [my_project_tree, shared_project_tree].each do |tree| %>
- <% tree.each do |projectnode| %>
- <% if projectnode[:object].is_a? String %>
- <div class="row" style="padding-left: <%= 1 + projectnode[:depth] %>em; margin-right: 0px">
- <i class="fa fa-fw fa-share-alt"></i>
- <%= projectnode[:object] %>
- </div>
- <% else
- row_selectable = !params[:editable] || projectnode[:object].editable?
- if projectnode[:object].uuid == current_user.uuid
- row_name = "Home"
- row_selectable = true
- else
- row_name = projectnode[:object].friendly_link_name || 'New project'
- end %>
- <div class="<%= 'selectable project' if row_selectable %> row"
- style="padding-left: <%= 1 + projectnode[:depth] %>em; margin-right: 0px" data-object-uuid="<%= projectnode[:object].uuid %>">
- <i class="fa fa-fw fa-folder-o"></i> <%= row_name %>
- </div>
- <% end %>
+ <% starred_projects = my_starred_projects current_user%>
+ <% if starred_projects.andand.any? %>
+ <% writable_projects = starred_projects.select(&:editable?) %>
+ <% writable_projects.each do |projectnode| %>
+ <% row_name = projectnode.friendly_link_name || 'New project' %>
+ <div class="selectable project row"
+ style="padding-left: 1em; margin-right: 0px"
+ data-object-uuid="<%= projectnode.uuid %>">
+ <i class="fa fa-fw fa-folder-o"></i> <%= row_name %> <i class="fa fa-fw fa-star"></i>
+ </div>
<% end %>
<% end %>
+
+ <% my_projects = my_wanted_projects_tree(current_user) %>
+ <% my_projects[0].each do |projectnode| %>
+ <% if projectnode[:object].uuid == current_user.uuid
+ row_name = "Home"
+ else
+ row_name = projectnode[:object].friendly_link_name || 'New project'
+ end %>
+ <div class="selectable project row"
+ style="padding-left: <%= 1 + projectnode[:depth] %>em; margin-right: 0px"
+ data-object-uuid="<%= projectnode[:object].uuid %>">
+ <i class="fa fa-fw fa-folder-o"></i> <%= row_name %>
+ </div>
+ <% end %>
</div>
+
+ <% if my_projects[1] or my_projects[2] or my_projects[0].size > 200 %>
+ <div>Some of your projects are omitted. Add projects of interest to favorites.</div>
+ <% end %>
</div>
<div class="modal-footer">
import cwltool.draft2tool
import cwltool.workflow
import cwltool.main
+from cwltool.process import shortname
import threading
import cwltool.docker
import fnmatch
args = [image_name]
if image_tag:
args.append(image_tag)
+ logger.info("Uploading Docker image %s", ":".join(args))
arvados.commands.keepdocker.main(args)
return dockerRequirement["dockerImageId"]
"script_version": "master",
"script_parameters": {"tasks": [script_parameters]},
"runtime_constraints": runtime_constraints
- }, find_or_create=kwargs.get("enable_reuse", True)).execute()
+ }, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
self.arvrunner.jobs[response["uuid"]] = self
- logger.info("Job %s is %s", response["uuid"], response["state"])
+ self.arvrunner.pipeline["components"][self.name] = {"job": response}
+ self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update(uuid=self.arvrunner.pipeline["uuid"],
+ body={
+ "components": self.arvrunner.pipeline["components"]
+ }).execute(num_retries=self.arvrunner.num_retries)
+
+ logger.info("Job %s (%s) is %s", self.name, response["uuid"], response["state"])
if response["state"] in ("Complete", "Failed", "Cancelled"):
self.done(response)
logger.error("Got error %s" % str(e))
self.output_callback({}, "permanentFail")
+ def update_pipeline_component(self, record):
+ self.arvrunner.pipeline["components"][self.name] = {"job": record}
+ self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().update(uuid=self.arvrunner.pipeline["uuid"],
+ body={
+ "components": self.arvrunner.pipeline["components"]
+ }).execute(num_retries=self.arvrunner.num_retries)
def done(self, record):
+ try:
+ self.update_pipeline_component(record)
+ except:
+ pass
+
try:
if record["state"] == "Complete":
processStatus = "success"
try:
outputs = {}
- outputs = self.collect_outputs("keep:" + record["output"])
+ if record["output"]:
+ outputs = self.collect_outputs("keep:" + record["output"])
except Exception as e:
logger.exception("Got exception while collecting job outputs:")
processStatus = "permanentFail"
self._pathmap[src] = (src, "$(task.keep)/%s" % src[5:])
if src not in self._pathmap:
ab = cwltool.pathmapper.abspath(src, basedir)
- st = arvados.commands.run.statfile("", ab)
+ st = arvados.commands.run.statfile("", ab, fnPattern="$(task.keep)/%s/%s")
if kwargs.get("conformance_test"):
self._pathmap[src] = (src, ab)
elif isinstance(st, arvados.commands.run.UploadFile):
self.cond = threading.Condition(self.lock)
self.final_output = None
self.uploaded = {}
+ self.num_retries = 4
def arvMakeTool(self, toolpath_object, **kwargs):
if "class" in toolpath_object and toolpath_object["class"] == "CommandLineTool":
def output_callback(self, out, processStatus):
if processStatus == "success":
logger.info("Overall job status is %s", processStatus)
+ self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
+ body={"state": "Complete"}).execute(num_retries=self.num_retries)
+
else:
logger.warn("Overall job status is %s", processStatus)
+ self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
+ body={"state": "Failed"}).execute(num_retries=self.num_retries)
self.final_output = out
+
def on_message(self, event):
if "object_uuid" in event:
if event["object_uuid"] in self.jobs and event["event_type"] == "update":
if event["properties"]["new_attributes"]["state"] == "Running" and self.jobs[event["object_uuid"]].running is False:
- logger.info("Job %s is Running", event["object_uuid"])
+ uuid = event["object_uuid"]
with self.lock:
- self.jobs[event["object_uuid"]].running = True
+ j = self.jobs[uuid]
+ logger.info("Job %s (%s) is Running", j.name, uuid)
+ j.running = True
+ j.update_pipeline_component(event["properties"]["new_attributes"])
elif event["properties"]["new_attributes"]["state"] in ("Complete", "Failed", "Cancelled"):
- logger.info("Job %s is %s", event["object_uuid"], event["properties"]["new_attributes"]["state"])
+ uuid = event["object_uuid"]
try:
self.cond.acquire()
- self.jobs[event["object_uuid"]].done(event["properties"]["new_attributes"])
+ j = self.jobs[uuid]
+ logger.info("Job %s (%s) is %s", j.name, uuid, event["properties"]["new_attributes"]["state"])
+ j.done(event["properties"]["new_attributes"])
self.cond.notify()
finally:
self.cond.release()
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
+ self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
+ "components": {},
+ "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
+
self.fs_access = CollectionFsAccess(input_basedir)
kwargs["fs_access"] = self.fs_access
'bin/arvados-cwl-runner'
],
install_requires=[
- 'cwltool>=1.0.20151026181844',
- 'arvados-python-client>=0.1.20151023214338'
+ 'cwltool>=1.0.20160129152024',
+ 'arvados-python-client>=0.1.20160122132348'
],
zip_safe=True,
cmdclass={'egg_info': tagger},
# ArvFile() (file already exists in a collection), UploadFile() (file needs to
# be uploaded to a collection), or simply returns prefix+fn (which yields the
# original parameter string).
-def statfile(prefix, fn):
+def statfile(prefix, fn, fnPattern="$(file %s/%s)", dirPattern="$(dir %s/%s/)"):
absfn = os.path.abspath(fn)
if os.path.exists(absfn):
st = os.stat(absfn)
sp = os.path.split(absfn)
(pdh, branch) = is_in_collection(sp[0], sp[1])
if pdh:
- return ArvFile(prefix, "$(file %s/%s)" % (pdh, branch))
+ return ArvFile(prefix, fnPattern % (pdh, branch))
else:
# trim leading '/' for path prefix test later
return UploadFile(prefix, absfn[1:])
sp = os.path.split(absfn)
(pdh, branch) = is_in_collection(sp[0], sp[1])
if pdh:
- return ArvFile(prefix, "$(dir %s/%s/)" % (pdh, branch))
+ return ArvFile(prefix, dirPattern % (pdh, branch))
return prefix+fn
return _subscribe_websocket(api, filters, on_event, last_log_id)
try:
- return _subscribe_websocket(api, filters, on_event, last_log_id)
+ if not config.flag_is_true('ARVADOS_DISABLE_WEBSOCKETS'):
+ return _subscribe_websocket(api, filters, on_event, last_log_id)
+ else:
+ _logger.info("Using polling because ARVADOS_DISABLE_WEBSOCKETS is true")
except Exception as e:
_logger.warn("Falling back to polling after websocket error: %s" % e)
p = PollClient(api, filters, on_event, poll_fallback, last_log_id)
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
+ 'google-api-python-client==1.4.2',
+ 'oauth2client >=1.4.6, <2',
'ciso8601',
- 'google-api-python-client',
'httplib2',
'pycurl >=7.19.5.1, <7.21.5',
'python-gflags<3.0',
# Dev/test SSL certificates
/self-signed.key
/self-signed.pem
+
+# Generated git-commit.version file
+/git-commit.version
projects / arvados.git / commitdiff