if hint[:keep_cache]
keep_cache = hint[:keep_cache]
end
+ if hint[:acrContainerImage]
+ attrs['container_image'] = hint[:acrContainerImage]
+ end
end
end
end
assert_text 'script version'
assert_no_selector 'a', text: 'Run this pipeline'
else
- within first('tr[data-kind="arvados#workflow"]') do
+ within 'tr[data-kind="arvados#workflow"]', text: "Workflow with default input specifications" do
click_link 'Show'
end
assert_text "This container request was created from the workflow"
assert_match /Provide a value for .* then click the \"Run\" button to start the workflow/, page.text
end
+
+ test "create workflow with WorkflowRunnerResources" do
+ visit page_with_token('active', '/workflows/zzzzz-7fd4e-validwithinput3')
+
+ find('a,button', text: 'Run this workflow').click
+
+ # Choose project for the container_request being created
+ within('.modal-dialog') do
+ find('.selectable', text: 'A Project').click
+ find('button', text: 'Choose').click
+ end
+ click_link 'Advanced'
+ click_link("API response")
+ assert_text('"container_image": "arvados/jobs:2.0.4"')
+ assert_text('"vcpus": 2')
+ assert_text('"ram": 1293942784')
+ assert_text('"--collection-cache-size=678"')
+
+ end
end
overquota = sorted[i:]
break tryrun
}
+ if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ logger.Info("not locking: crunch-run process from previous attempt has not exited")
+ continue
+ }
go sch.lockContainer(logger, ctr.UUID)
unalloc[it]--
case arvados.ContainerStateLocked:
// a higher-priority container on the
// same instance type. Don't let this
// one sneak in ahead of it.
- } else if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ } else if sch.pool.KillContainer(ctr.UUID, "about to start") {
logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
} else if sch.pool.StartContainer(it, ctr) {
// Success.
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder)
+ make_builder, arvados_jobs_image)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
from .perf import Perf
sum_res_pars = ("outdirMin", "outdirMax")
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
- submit_runner_ram=0, name=None, merged_map=None):
+ submit_runner_ram=0, name=None, merged_map=None,
+ submit_runner_image=None):
packed = packed_workflow(arvRunner, tool, merged_map)
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
+ wf_runner_resources = None
+
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ wf_runner_resources = h
+ found = True
+ break
+ if not found:
+ wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+ hints.append(wf_runner_resources)
+
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
if submit_runner_ram:
- hints = main.get("hints", [])
- found = False
- for h in hints:
- if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
- h["ramMin"] = submit_runner_ram
- found = True
- break
- if not found:
- hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
- "ramMin": submit_runner_ram})
- main["hints"] = hints
+ wf_runner_resources["ramMin"] = submit_runner_ram
+
+ main["hints"] = hints
body = {
"workflow": {
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
- merged_map=merged_map),
+ merged_map=merged_map,
+ submit_runner_image=runtimeContext.submit_runner_image),
"success")
self.apply_reqs(job_order, tool)
#
# SPDX-License-Identifier: Apache-2.0
+fpm_depends+=(nodejs)
+
case "$TARGET" in
- debian8)
- fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
- ;;
debian9 | ubuntu1604)
fpm_depends+=(libcurl3-gnutls)
;;
"$graph": [
{
"class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
"id": "#main",
"inputs": [],
"outputs": [],
}
],
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
- open("tests/wf/expect_packed.cwl").read())
+ open("tests/wf/expect_upload_packed.cwl").read().rstrip())
@stubs
def test_create(self, stubs):
stubs.capture_stdout, sys.stderr, api_client=stubs.api)
toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
- expect_workflow = StripYAMLComments(open(toolfile).read())
+ expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
body = {
"workflow": {
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "baseCommand": "cat",
+ "class": "CommandLineTool",
+ "id": "#submit_tool.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt"
+ },
+ "id": "#submit_tool.cwl/x",
+ "inputBinding": {
+ "position": 1
+ },
+ "type": "File"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/x",
+ "source": "#main/x"
+ }
+ ],
+ "out": [],
+ "run": "#submit_tool.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
id: ex_string_def
default: hello-testing-123
outputs: []
+
+workflow_with_wrr:
+ uuid: zzzzz-7fd4e-validwithinput3
+ owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
+ name: Workflow with WorkflowRunnerResources
+ description: this workflow has WorkflowRunnerResources
+ created_at: <%= 1.minute.ago.to_s(:db) %>
+ definition: |
+ cwlVersion: v1.0
+ class: CommandLineTool
+ hints:
+ - class: http://arvados.org/cwl#WorkflowRunnerResources
+ acrContainerImage: arvados/jobs:2.0.4
+ ramMin: 1234
+ coresMin: 2
+ keep_cache: 678
+ baseCommand:
+ - echo
+ inputs:
+ - type: string
+ id: ex_string
+ - type: string
+ id: ex_string_def
+ default: hello-testing-123
+ outputs: []
logins = arv.virtual_machine.logins(:uuid => vm_uuid)[:items]
logins = [] if logins.nil?
- logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:virtual_machine_uuid] != vm_uuid }
+ logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:virtual_machine_uuid] != vm_uuid }
# No system users
uid_min = 1000
logins.each do |l|
keys[l[:username]] = Array.new() if not keys.has_key?(l[:username])
key = l[:public_key]
- # Handle putty-style ssh public keys
- key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
- key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
- key.gsub!(/\n/,'')
- key.strip
-
- keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ if !key.nil?
+ # Handle putty-style ssh public keys
+ key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ key.gsub!(/\n/,'')
+ key.strip
+
+ keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ end
end
seen = Hash.new()
fi
}
+listusers() {
+ docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+}
+
wait_for_arvbox() {
FF=/tmp/arvbox-fifo-$$
mkfifo $FF
wait_for_arvbox
echo "The Arvados source code is checked out at: $ARVADOS_ROOT"
echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem"
+ if [[ "$(listusers)" =~ ^\{\} ]] ; then
+ echo "No users defined, use 'arvbox adduser' to add user logins"
+ else
+ echo "Use 'arvbox listusers' to see user logins"
+ fi
else
echo "Unknown configuration '$CONFIG'"
fi
;;
listusers)
- exec docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+ listusers
;;
*)
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
"http://localhost:${services[controller]}": {}
- RailsAPI:
- InternalURLs:
- "http://localhost:${services[api]}/": {}
PostgreSQL:
ConnectionPool: 32 # max concurrent connections per arvados server daemon
Connection:
projects / arvados.git / commitdiff