From 8acbf6d96287dbdf3ac69a4b0ef98452e4c203ed Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 16 Nov 2018 16:15:11 -0500 Subject: [PATCH] 14198: Federation feature and testing * Evaluate ClusterTarget in the job step that declares it * Don't use get_requirement to inherit ClusterTarget. * Workflow to start and configure a federation of N arvboxes, initialize test user * Test cases. * Prepare/scrub before running test * Use --always-submit-runner * Tests use isolated inputs. Return success boolean Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- .licenseignore | 1 + sdk/cwl/arvados_cwl/arvtool.py | 33 ++-- sdk/cwl/arvados_cwl/arvworkflow.py | 8 +- sdk/cwl/tests/federation/README | 55 +++++++ .../federation/arvbox/arvbox-fed-config.cwl | 66 ++++++++ .../tests/federation/arvbox/arvbox-fed.cwl | 72 +++++++++ .../tests/federation/arvbox/arvbox-main.cwl | 38 +++++ .../tests/federation/arvbox/arvbox-mkdir.cwl | 47 ++++++ .../federation/arvbox/arvbox-setup-user.cwl | 34 +++++ .../tests/federation/arvbox/arvbox-start.cwl | 72 +++++++++ .../tests/federation/arvbox/arvbox-stop.cwl | 17 +++ sdk/cwl/tests/federation/cases/base-case.cwl | 29 ++++ sdk/cwl/tests/federation/cases/md5sum.cwl | 21 +++ .../cases/runner-home-step-remote.cwl | 29 ++++ .../cases/runner-remote-step-home.cwl | 29 ++++ .../tests/federation/data/base-case-input.txt | 16 ++ .../data/runner-home-step-remote-input.txt | 16 ++ .../data/runner-remote-step-home-input.txt | 16 ++ .../federation/framework/check-exist.cwl | 42 +++++ .../tests/federation/framework/check_exist.py | 25 +++ .../federation/framework/dockerbuild.cwl | 21 +++ .../tests/federation/framework/prepare.cwl | 48 ++++++ sdk/cwl/tests/federation/framework/prepare.py | 42 +++++ .../tests/federation/framework/run-acr.cwl | 54 +++++++ .../tests/federation/framework/setup_user.py | 40 +++++ .../tests/federation/framework/testcase.cwl | 77 ++++++++++ sdk/cwl/tests/federation/main.cwl | 143 ++++++++++++++++++ 27 files changed, 1076 insertions(+), 15 deletions(-) create mode 100644 sdk/cwl/tests/federation/README create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-main.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-start.cwl create mode 100644 sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl create mode 100644 sdk/cwl/tests/federation/cases/base-case.cwl create mode 100644 sdk/cwl/tests/federation/cases/md5sum.cwl create mode 100644 sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl create mode 100644 sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl create mode 100644 sdk/cwl/tests/federation/data/base-case-input.txt create mode 100644 sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt create mode 100644 sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt create mode 100644 sdk/cwl/tests/federation/framework/check-exist.cwl create mode 100644 sdk/cwl/tests/federation/framework/check_exist.py create mode 100644 sdk/cwl/tests/federation/framework/dockerbuild.cwl create mode 100644 sdk/cwl/tests/federation/framework/prepare.cwl create mode 100644 sdk/cwl/tests/federation/framework/prepare.py create mode 100644 sdk/cwl/tests/federation/framework/run-acr.cwl create mode 100644 sdk/cwl/tests/federation/framework/setup_user.py create mode 100644 sdk/cwl/tests/federation/framework/testcase.cwl create mode 100755 sdk/cwl/tests/federation/main.cwl diff --git a/.licenseignore b/.licenseignore index 51a1e7cbd2..113bf4fa4e 100644 --- a/.licenseignore +++ b/.licenseignore @@ -46,6 +46,7 @@ docker/jobs/apt.arvados.org.list */script/rails sdk/cwl/tests/input/blorp.txt sdk/cwl/tests/tool/blub.txt +sdk/cwl/tests/federation/data/* sdk/go/manifest/testdata/*_manifest sdk/java/.classpath sdk/java/pom.xml diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py index 1a0c81b283..ee9dd25a13 100644 --- a/sdk/cwl/arvados_cwl/arvtool.py +++ b/sdk/cwl/arvados_cwl/arvtool.py @@ -10,16 +10,26 @@ from functools import partial from schema_salad.sourceline import SourceLine from cwltool.errors import WorkflowException -def check_cluster_target(self, builder, runtimeContext): - cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget") - if cluster_target_req and runtimeContext.cluster_target_id != id(cluster_target_req): - with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug): - runtimeContext.cluster_target_id = id(cluster_target_req) - runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster - runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid - if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in self.arvrunner.api._rootDesc["remoteHosts"]: - raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster, - ", ".join(self.arvrunner.api._rootDesc["remoteHosts"].keys()))) +def set_cluster_target(tool, arvrunner, builder, runtimeContext): + cluster_target_req = None + for field in ("hints", "requirements"): + if field not in tool: + continue + for item in tool[field]: + if item["class"] == "http://arvados.org/cwl#ClusterTarget": + cluster_target_req = item + + if cluster_target_req is None: + return runtimeContext + + with SourceLine(cluster_target_req, None, WorkflowException, runtimeContext.debug): + runtimeContext = runtimeContext.copy() + runtimeContext.submit_runner_cluster = builder.do_eval(cluster_target_req.get("cluster_id")) or runtimeContext.submit_runner_cluster + runtimeContext.project_uuid = builder.do_eval(cluster_target_req.get("project_uuid")) or runtimeContext.project_uuid + if runtimeContext.submit_runner_cluster and runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"]: + raise WorkflowException("Unknown or invalid cluster id '%s' known clusters are %s" % (runtimeContext.submit_runner_cluster, + ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys()))) + return runtimeContext class ArvadosCommandTool(CommandLineTool): """Wrap cwltool CommandLineTool to override selected methods.""" @@ -56,8 +66,7 @@ class ArvadosCommandTool(CommandLineTool): joborder = builder.job runtimeContext = runtimeContext.copy() - - check_cluster_target(self, builder, runtimeContext) + runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) if runtimeContext.work_api == "containers": dockerReq, is_req = self.get_requirement("DockerRequirement") diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py index b26bee8247..2180066338 100644 --- a/sdk/cwl/arvados_cwl/arvworkflow.py +++ b/sdk/cwl/arvados_cwl/arvworkflow.py @@ -22,7 +22,7 @@ import ruamel.yaml as yaml from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields, discover_secondary_files) from .pathmapper import ArvPathMapper, trim_listing -from .arvtool import ArvadosCommandTool, check_cluster_target +from .arvtool import ArvadosCommandTool, set_cluster_target from .perf import Perf logger = logging.getLogger('arvados.cwl-runner') @@ -134,9 +134,10 @@ class ArvadosWorkflowStep(WorkflowStep): def job(self, joborder, output_callback, runtimeContext): builder = self._init_job({shortname(k): v for k,v in joborder.items()}, runtimeContext) - check_cluster_target(self, builder, runtimeContext) + runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext) + class ArvadosWorkflow(Workflow): """Wrap cwltool Workflow to override selected methods.""" @@ -148,11 +149,12 @@ class ArvadosWorkflow(Workflow): self.wf_reffiles = [] self.loadingContext = loadingContext super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext) + self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget") def job(self, joborder, output_callback, runtimeContext): builder = self._init_job(joborder, runtimeContext) - check_cluster_target(self, builder, runtimeContext) + runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext) req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer") if not req: diff --git a/sdk/cwl/tests/federation/README b/sdk/cwl/tests/federation/README new file mode 100644 index 0000000000..e537939006 --- /dev/null +++ b/sdk/cwl/tests/federation/README @@ -0,0 +1,55 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +Things to test. + +Single step --submit --no-wait workflow, matrix of: + +Runner on home/remote cluster +Docker image on home/remote cluster +Step on home/remote cluster + +Two step workflow, matrix of: + +Step1 on home cluster -> Step2 on remote cluster +Step1 on remote cluster -> Step2 on home cluster + +Three step workflow: + +(Step1 on remoteA cluster, Step2 on remoteB cluster) -> Step3 on home cluster +Step1 on remoteA cluster -> Step2 on remoteB cluster -> Step3 on home cluster + +Workflow which has a remote collection in InitialWorkDir, which is captured in output. + +Workflow with file input that has a secondary file in a separate +collection, which is remote. + + +Need to pull Docker image +Can't run in container because it doesn't have access to Docker from inside -> how can we workaround? +Turns out we also can't run in container because cwltool doesn't pathmap inside "Any" (this is a bug -> fix this) +Try setting up a virtualenv -> annoying because you need C dependencies to build +Virtualenv sets up shop in /tmp instead of the real directory +--relocatable ??? + + +arvados/jobs doesn't have the docker client +arvbox has docker client + +- Still need venv or something to run cwltool to manage all this. +- Unless we run cwltool in docker as well. + +Can use cwl-docker.sh to run cwltool in docker + +- arvbox needs Docker. This will *probably* work launched from container if base dir is under PWD +- want to run arvados-cwl-runner. this needs Docker to pull images. + +For a-c-r solutions are one of: + +1) Don't run in Docker, use virtualenv instead. +2) Access to Docker from inside Docker to pull images. +2a) Docker-in-Docker +2b) Bind mount Docker socket +2c) Communicate to Docker daemon over TCP +3) Arvados feature to pull images diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl new file mode 100644 index 0000000000..77567ee89d --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-fed-config.cwl @@ -0,0 +1,66 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +inputs: + container_name: string + this_cluster_id: string + cluster_ids: string[] + cluster_hosts: string[] + arvbox_data: Directory +outputs: + arvbox_data_out: + type: Directory + outputBinding: + outputEval: $(inputs.arvbox_data) +requirements: + EnvVarRequirement: + envDef: + ARVBOX_CONTAINER: $(inputs.container_name) + ARVBOX_DATA: $(inputs.arvbox_data.path) + InitialWorkDirRequirement: + listing: + - entryname: cluster_config.yml.override + entry: >- + ${ + var remoteClusters = {}; + for (var i = 0; i < inputs.cluster_ids.length; i++) { + remoteClusters[inputs.cluster_ids[i]] = { + "Host": inputs.cluster_hosts[i], + "Proxy": true, + "Insecure": true + }; + } + var r = {"Clusters": {}}; + r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters}; + return JSON.stringify(r); + } + - entryname: application.yml.override + entry: >- + ${ + var remoteClusters = {}; + for (var i = 0; i < inputs.cluster_ids.length; i++) { + remoteClusters[inputs.cluster_ids[i]] = inputs.cluster_hosts[i]; + } + return JSON.stringify({"development": {"remote_hosts": remoteClusters}}); + } + cwltool:LoadListingRequirement: + loadListing: no_listing + ShellCommandRequirement: {} + InlineJavascriptRequirement: {} + cwltool:InplaceUpdateRequirement: + inplaceUpdate: true +arguments: + - shellQuote: false + valueFrom: | + docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados + docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config + arvbox sv restart api + arvbox sv restart controller + arvbox sv restart keepstore0 + arvbox sv restart keepstore1 diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl new file mode 100644 index 0000000000..91e6220220 --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-fed.cwl @@ -0,0 +1,72 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +requirements: + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + cwltool:LoadListingRequirement: + loadListing: no_listing + InlineJavascriptRequirement: {} +inputs: + containers: + type: string[] + default: [fedbox1, fedbox2, fedbox3] + arvbox_base: Directory + in_acr: string? + insecure: + type: boolean + default: true +outputs: + arvados_api_token: + type: string + outputSource: setup-user/test_user_token + arvados_api_hosts: + type: string[] + outputSource: start/container_host + arvados_cluster_ids: + type: string[] + outputSource: start/cluster_id + acr: + type: string? + outputSource: in_acr + arvado_api_host_insecure: + type: boolean + outputSource: insecure +steps: + mkdir: + in: + containers: containers + arvbox_base: arvbox_base + out: [arvbox_data] + run: arvbox-mkdir.cwl + start: + in: + container_name: containers + arvbox_data: mkdir/arvbox_data + out: [cluster_id, container_host, arvbox_data_out, superuser_token] + scatter: [container_name, arvbox_data] + scatterMethod: dotproduct + run: arvbox-start.cwl + fed-config: + in: + container_name: containers + this_cluster_id: start/cluster_id + cluster_ids: start/cluster_id + cluster_hosts: start/container_host + arvbox_data: start/arvbox_data_out + out: [] + scatter: [container_name, this_cluster_id, arvbox_data] + scatterMethod: dotproduct + run: arvbox-fed-config.cwl + setup-user: + in: + container_host: {source: start/container_host, valueFrom: "$(self[0])"} + superuser_token: {source: start/superuser_token, valueFrom: "$(self[0])"} + out: [test_user_uuid, test_user_token] + run: arvbox-setup-user.cwl diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl new file mode 100644 index 0000000000..a6fd008237 --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-main.cwl @@ -0,0 +1,38 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +requirements: + cwltool:LoadListingRequirement: + loadListing: no_listing + SubworkflowFeatureRequirement: {} +inputs: + arvbox_base: Directory + acr: string? +outputs: [] +steps: + run-arvbox: + in: + containers: + default: [fedbox1, fedbox2, fedbox3] + arvbox_base: arvbox_base + out: [cluster_ids, container_hosts, test_user_uuid, test_user_token] + run: arvbox-fed.cwl + run-main: + in: + arvados_api_host_home: {source: run-arvbox/container_hosts, valueFrom: "$(self[0])"} + arvados_home_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[0])"} + arvados_api_token: run-arvbox/test_user_token + arvado_api_host_insecure: {default: true} + arvados_api_host_clusterB: {source: run-arvbox/container_hosts, valueFrom: "$(self[1])"} + arvados_clusterB_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[1])"} + arvados_api_host_clusterC: {source: run-arvbox/container_hosts, valueFrom: "$(self[2])"} + arvados_clusterC_id: {source: run-arvbox/cluster_ids, valueFrom: "$(self[2])"} + acr: acr + out: [base-case-out, runner-home-step-remote-out] + run: main.cwl diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl new file mode 100644 index 0000000000..727d491a38 --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-mkdir.cwl @@ -0,0 +1,47 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +inputs: + containers: + type: + type: array + items: string + inputBinding: + position: 3 + valueFrom: | + ${ + return "base/"+self; + } + arvbox_base: Directory +outputs: + arvbox_data: + type: Directory[] + outputBinding: + glob: | + ${ + var r = []; + for (var i = 0; i < inputs.containers.length; i++) { + r.push("base/"+inputs.containers[i]); + } + return r; + } +requirements: + InitialWorkDirRequirement: + listing: + - entry: $(inputs.arvbox_base) + entryname: base + writable: true + cwltool:LoadListingRequirement: + loadListing: no_listing + InlineJavascriptRequirement: {} + cwltool:InplaceUpdateRequirement: + inplaceUpdate: true +arguments: + - mkdir + - "-p" diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl new file mode 100644 index 0000000000..0fddc1b879 --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-setup-user.cwl @@ -0,0 +1,34 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +requirements: + EnvVarRequirement: + envDef: + ARVADOS_API_HOST: $(inputs.container_host) + ARVADOS_API_TOKEN: $(inputs.superuser_token) + ARVADOS_API_HOST_INSECURE: "true" + cwltool:LoadListingRequirement: + loadListing: no_listing + InlineJavascriptRequirement: {} + cwltool:InplaceUpdateRequirement: + inplaceUpdate: true + DockerRequirement: + dockerPull: arvados/jobs +inputs: + container_host: string + superuser_token: string + make_user_script: + type: File + default: + class: File + location: setup_user.py +outputs: + test_user_uuid: string + test_user_token: string +arguments: [python2, $(inputs.make_user_script)] \ No newline at end of file diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl new file mode 100644 index 0000000000..f69775a530 --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-start.cwl @@ -0,0 +1,72 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +inputs: + container_name: string + arvbox_data: Directory +outputs: + cluster_id: + type: string + outputBinding: + glob: status.txt + loadContents: true + outputEval: | + ${ + var sp = self[0].contents.split("\n"); + for (var i = 0; i < sp.length; i++) { + if (sp[i].startsWith("Cluster id: ")) { + return sp[i].substr(12); + } + } + } + container_host: + type: string + outputBinding: + glob: status.txt + loadContents: true + outputEval: | + ${ + var sp = self[0].contents.split("\n"); + for (var i = 0; i < sp.length; i++) { + if (sp[i].startsWith("Container IP: ")) { + return sp[i].substr(14)+":8000"; + } + } + } + superuser_token: + type: string + outputBinding: + glob: superuser_token.txt + loadContents: true + outputEval: $(self[0].contents.trim()) + arvbox_data_out: + type: Directory + outputBinding: + outputEval: $(inputs.arvbox_data) +requirements: + EnvVarRequirement: + envDef: + ARVBOX_CONTAINER: $(inputs.container_name) + ARVBOX_DATA: $(inputs.arvbox_data.path) + ShellCommandRequirement: {} + InitialWorkDirRequirement: + listing: + - entry: $(inputs.arvbox_data) + entryname: $(inputs.container_name) + writable: true + cwltool:InplaceUpdateRequirement: + inplaceUpdate: true + InlineJavascriptRequirement: {} +arguments: + - shellQuote: false + valueFrom: | + set -e + arvbox start dev + arvbox status > status.txt + arvbox cat /var/lib/arvados/superuser_token > superuser_token.txt \ No newline at end of file diff --git a/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl b/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl new file mode 100644 index 0000000000..2ea4c0f74f --- /dev/null +++ b/sdk/cwl/tests/federation/arvbox/arvbox-stop.cwl @@ -0,0 +1,17 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +inputs: + container_name: string +outputs: [] +requirements: + EnvVarRequirement: + envDef: + ARVBOX_CONTAINER: $(inputs.container_name) +arguments: [arvbox, stop] \ No newline at end of file diff --git a/sdk/cwl/tests/federation/cases/base-case.cwl b/sdk/cwl/tests/federation/cases/base-case.cwl new file mode 100644 index 0000000000..2e0ff64c6b --- /dev/null +++ b/sdk/cwl/tests/federation/cases/base-case.cwl @@ -0,0 +1,29 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" +requirements: + InlineJavascriptRequirement: {} + DockerRequirement: + dockerPull: arvados/fed-test:base-case + arv:ClusterTarget: + cluster_id: $(inputs.runOnCluster) +inputs: + inp: + type: File + inputBinding: {} + runOnCluster: string +outputs: + hash: + type: File + outputSource: md5sum/hash +steps: + md5sum: + in: + inp: inp + out: [hash] + run: md5sum.cwl \ No newline at end of file diff --git a/sdk/cwl/tests/federation/cases/md5sum.cwl b/sdk/cwl/tests/federation/cases/md5sum.cwl new file mode 100644 index 0000000000..af119990a3 --- /dev/null +++ b/sdk/cwl/tests/federation/cases/md5sum.cwl @@ -0,0 +1,21 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +$namespaces: + arv: "http://arvados.org/cwl#" +requirements: + InlineJavascriptRequirement: {} +inputs: + inp: + type: File +outputs: + hash: + type: File + outputBinding: + glob: out.txt +stdin: $(inputs.inp.path) +stdout: out.txt +arguments: ["md5sum", "-"] diff --git a/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl new file mode 100644 index 0000000000..70d0176390 --- /dev/null +++ b/sdk/cwl/tests/federation/cases/runner-home-step-remote.cwl @@ -0,0 +1,29 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" +requirements: + InlineJavascriptRequirement: {} + DockerRequirement: + dockerPull: arvados/fed-test:runner-home-step-remote + arv:ClusterTarget: + cluster_id: $(inputs.runOnCluster) +inputs: + inp: + type: File + inputBinding: {} + runOnCluster: string +outputs: + hash: + type: File + outputSource: md5sum/hash +steps: + md5sum: + in: + inp: inp + out: [hash] + run: md5sum.cwl \ No newline at end of file diff --git a/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl new file mode 100644 index 0000000000..33681f18e6 --- /dev/null +++ b/sdk/cwl/tests/federation/cases/runner-remote-step-home.cwl @@ -0,0 +1,29 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" +requirements: + InlineJavascriptRequirement: {} + DockerRequirement: + dockerPull: arvados/fed-test:runner-remote-step-home + arv:ClusterTarget: + cluster_id: $(inputs.runOnCluster) +inputs: + inp: + type: File + inputBinding: {} + runOnCluster: string +outputs: + hash: + type: File + outputSource: md5sum/hash +steps: + md5sum: + in: + inp: inp + out: [hash] + run: md5sum.cwl \ No newline at end of file diff --git a/sdk/cwl/tests/federation/data/base-case-input.txt b/sdk/cwl/tests/federation/data/base-case-input.txt new file mode 100644 index 0000000000..761b840cd1 --- /dev/null +++ b/sdk/cwl/tests/federation/data/base-case-input.txt @@ -0,0 +1,16 @@ +Call me base-case. Some years ago--never mind how long precisely--having +little or no money in my purse, and nothing particular to interest me on +shore, I thought I would sail about a little and see the watery part of +the world. It is a way I have of driving off the spleen and regulating +the circulation. Whenever I find myself growing grim about the mouth; +whenever it is a damp, drizzly November in my soul; whenever I find +myself involuntarily pausing before coffin warehouses, and bringing up +the rear of every funeral I meet; and especially whenever my hypos get +such an upper hand of me, that it requires a strong moral principle to +prevent me from deliberately stepping into the street, and methodically +knocking people's hats off--then, I account it high time to get to sea +as soon as I can. This is my substitute for pistol and ball. With a +philosophical flourish Cato throws himself upon his sword; I quietly +take to the ship. There is nothing surprising in this. If they but knew +it, almost all men in their degree, some time or other, cherish very +nearly the same feelings towards the ocean with me. diff --git a/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt new file mode 100644 index 0000000000..91ab77d12d --- /dev/null +++ b/sdk/cwl/tests/federation/data/runner-home-step-remote-input.txt @@ -0,0 +1,16 @@ +Call me runner-home-step-remote. Some years ago--never mind how long precisely--having +little or no money in my purse, and nothing particular to interest me on +shore, I thought I would sail about a little and see the watery part of +the world. It is a way I have of driving off the spleen and regulating +the circulation. Whenever I find myself growing grim about the mouth; +whenever it is a damp, drizzly November in my soul; whenever I find +myself involuntarily pausing before coffin warehouses, and bringing up +the rear of every funeral I meet; and especially whenever my hypos get +such an upper hand of me, that it requires a strong moral principle to +prevent me from deliberately stepping into the street, and methodically +knocking people's hats off--then, I account it high time to get to sea +as soon as I can. This is my substitute for pistol and ball. With a +philosophical flourish Cato throws himself upon his sword; I quietly +take to the ship. There is nothing surprising in this. If they but knew +it, almost all men in their degree, some time or other, cherish very +nearly the same feelings towards the ocean with me. diff --git a/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt new file mode 100644 index 0000000000..e5673b848b --- /dev/null +++ b/sdk/cwl/tests/federation/data/runner-remote-step-home-input.txt @@ -0,0 +1,16 @@ +Call me runner-remote-step-home. Some years ago--never mind how long precisely--having +little or no money in my purse, and nothing particular to interest me on +shore, I thought I would sail about a little and see the watery part of +the world. It is a way I have of driving off the spleen and regulating +the circulation. Whenever I find myself growing grim about the mouth; +whenever it is a damp, drizzly November in my soul; whenever I find +myself involuntarily pausing before coffin warehouses, and bringing up +the rear of every funeral I meet; and especially whenever my hypos get +such an upper hand of me, that it requires a strong moral principle to +prevent me from deliberately stepping into the street, and methodically +knocking people's hats off--then, I account it high time to get to sea +as soon as I can. This is my substitute for pistol and ball. With a +philosophical flourish Cato throws himself upon his sword; I quietly +take to the ship. There is nothing surprising in this. If they but knew +it, almost all men in their degree, some time or other, cherish very +nearly the same feelings towards the ocean with me. diff --git a/sdk/cwl/tests/federation/framework/check-exist.cwl b/sdk/cwl/tests/federation/framework/check-exist.cwl new file mode 100644 index 0000000000..ebb0fb220f --- /dev/null +++ b/sdk/cwl/tests/federation/framework/check-exist.cwl @@ -0,0 +1,42 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +requirements: + InitialWorkDirRequirement: + listing: + - entryname: config.json + entry: |- + ${ + return JSON.stringify({ + check_collections: inputs.check_collections + }); + } + EnvVarRequirement: + envDef: + ARVADOS_API_HOST: $(inputs.arvados_api_host) + ARVADOS_API_TOKEN: $(inputs.arvados_api_token) + ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure) + InlineJavascriptRequirement: {} +inputs: + arvados_api_token: string + arvado_api_host_insecure: boolean + arvados_api_host: string + check_collections: string[] + preparescript: + type: File + default: + class: File + location: check_exist.py + inputBinding: + position: 1 +outputs: + success: + type: boolean + outputBinding: + glob: success + loadContents: true + outputEval: $(self[0].contents=="true") +baseCommand: python2 \ No newline at end of file diff --git a/sdk/cwl/tests/federation/framework/check_exist.py b/sdk/cwl/tests/federation/framework/check_exist.py new file mode 100644 index 0000000000..b3338939ed --- /dev/null +++ b/sdk/cwl/tests/federation/framework/check_exist.py @@ -0,0 +1,25 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import arvados +import json + +api = arvados.api() + +with open("config.json") as f: + config = json.load(f) + +success = True +for c in config["check_collections"]: + try: + api.collections().get(uuid=c).execute() + except Exception as e: + print("Checking for %s got exception %s" % (c, e)) + success = False + +with open("success", "w") as f: + if success: + f.write("true") + else: + f.write("false") diff --git a/sdk/cwl/tests/federation/framework/dockerbuild.cwl b/sdk/cwl/tests/federation/framework/dockerbuild.cwl new file mode 100644 index 0000000000..d00b3e2a51 --- /dev/null +++ b/sdk/cwl/tests/federation/framework/dockerbuild.cwl @@ -0,0 +1,21 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +inputs: + testcase: string +outputs: + imagename: + type: string + outputBinding: + outputEval: $(inputs.testcase) +requirements: + InitialWorkDirRequirement: + listing: + - entryname: Dockerfile + entry: |- + FROM debian@sha256:0a5fcee6f52d5170f557ee2447d7a10a5bdcf715dd7f0250be0b678c556a501b + LABEL org.arvados.testcase="$(inputs.testcase)" +arguments: [docker, build, -t, $(inputs.testcase), "."] diff --git a/sdk/cwl/tests/federation/framework/prepare.cwl b/sdk/cwl/tests/federation/framework/prepare.cwl new file mode 100644 index 0000000000..03f792c5e1 --- /dev/null +++ b/sdk/cwl/tests/federation/framework/prepare.cwl @@ -0,0 +1,48 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +requirements: + InitialWorkDirRequirement: + listing: + - entryname: input.json + entry: $(JSON.stringify(inputs.obj)) + - entryname: config.json + entry: |- + ${ + return JSON.stringify({ + arvados_cluster_ids: inputs.arvados_cluster_ids, + scrub_images: [inputs.scrub_image], + scrub_collections: inputs.scrub_collections + }); + } + EnvVarRequirement: + envDef: + ARVADOS_API_HOST: $(inputs.arvados_api_host) + ARVADOS_API_TOKEN: $(inputs.arvados_api_token) + ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure) + InlineJavascriptRequirement: {} +inputs: + arvados_api_token: string + arvado_api_host_insecure: boolean + arvados_api_host: string + arvados_cluster_ids: string[] + wf: File + obj: Any + scrub_image: string + scrub_collections: string[] + preparescript: + type: File + default: + class: File + location: prepare.py + inputBinding: + position: 1 +outputs: + done: + type: boolean + outputBinding: + outputEval: $(true) +baseCommand: python2 \ No newline at end of file diff --git a/sdk/cwl/tests/federation/framework/prepare.py b/sdk/cwl/tests/federation/framework/prepare.py new file mode 100644 index 0000000000..94a6a75edb --- /dev/null +++ b/sdk/cwl/tests/federation/framework/prepare.py @@ -0,0 +1,42 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import arvados +import json + +api = arvados.api() + +with open("config.json") as f: + config = json.load(f) + +scrub_collections = set(config["scrub_collections"]) + +for cluster_id in config["arvados_cluster_ids"]: + images = [] + for scrub_image in config["scrub_images"]: + sp = scrub_image.split(":") + image_name = sp[0] + image_tag = sp[1] if len(sp) > 1 else "latest" + images.append('{}:{}'.format(image_name, image_tag)) + + search_links = api.links().list( + filters=[['link_class', '=', 'docker_image_repo+tag'], + ['name', 'in', images]], + cluster_id=cluster_id).execute() + + head_uuids = [lk["head_uuid"] for lk in search_links["items"]] + cols = api.collections().list(filters=[["uuid", "in", head_uuids]], + cluster_id=cluster_id).execute() + for c in cols["items"]: + scrub_collections.add(c["portable_data_hash"]) + for lk in search_links["items"]: + api.links().delete(uuid=lk["uuid"]).execute() + + +for cluster_id in config["arvados_cluster_ids"]: + matches = api.collections().list(filters=[["portable_data_hash", "in", list(scrub_collections)]], + select=["uuid", "portable_data_hash"], cluster_id=cluster_id).execute() + for m in matches["items"]: + api.collections().delete(uuid=m["uuid"]).execute() + print("Scrubbed %s (%s)" % (m["uuid"], m["portable_data_hash"])) diff --git a/sdk/cwl/tests/federation/framework/run-acr.cwl b/sdk/cwl/tests/federation/framework/run-acr.cwl new file mode 100644 index 0000000000..b720eaa12f --- /dev/null +++ b/sdk/cwl/tests/federation/framework/run-acr.cwl @@ -0,0 +1,54 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: CommandLineTool +inputs: + acr: + type: string? + default: arvados-cwl-runner + inputBinding: + position: 1 + arvados_api_host: string + arvados_api_token: string + arvado_api_host_insecure: + type: boolean + default: false + runner_cluster: + type: string? + inputBinding: + prefix: --submit-runner-cluster + position: 2 + wf: + type: File + inputBinding: + position: 3 + obj: Any +requirements: + InitialWorkDirRequirement: + listing: + - entryname: input.json + entry: $(JSON.stringify(inputs.obj)) + EnvVarRequirement: + envDef: + ARVADOS_API_HOST: $(inputs.arvados_api_host) + ARVADOS_API_TOKEN: $(inputs.arvados_api_token) + ARVADOS_API_HOST_INSECURE: $(""+inputs.arvado_api_host_insecure) + InlineJavascriptRequirement: {} +outputs: + out: + type: Any + outputBinding: + glob: output.json + loadContents: true + #outputEval: $(JSON.parse(self[0].contents)) + outputEval: $(self[0].contents) +stdout: output.json +arguments: + - valueFrom: --disable-reuse + position: 2 + - valueFrom: --always-submit-runner + position: 2 + - valueFrom: input.json + position: 4 \ No newline at end of file diff --git a/sdk/cwl/tests/federation/framework/setup_user.py b/sdk/cwl/tests/federation/framework/setup_user.py new file mode 100644 index 0000000000..a456976bee --- /dev/null +++ b/sdk/cwl/tests/federation/framework/setup_user.py @@ -0,0 +1,40 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +import arvados +import arvados.errors +import time +import json + +while True: + try: + api = arvados.api() + break + except arvados.errors.ApiError: + time.sleep(2) + +existing = api.users().list(filters=[["email", "=", "test@example.com"], + ["is_active", "=", True]], limit=1).execute() +if existing["items"]: + u = existing["items"][0] +else: + u = api.users().create(body={ + 'first_name': 'Test', + 'last_name': 'User', + 'email': 'test@example.com', + 'is_admin': False + }).execute() + api.users().activate(uuid=u["uuid"]).execute() + +tok = api.api_client_authorizations().create(body={ + "api_client_authorization": { + "owner_uuid": u["uuid"] + } +}).execute() + +with open("cwl.output.json", "w") as f: + json.dump({ + "test_user_uuid": u["uuid"], + "test_user_token": "v2/%s/%s" % (tok["uuid"], tok["api_token"]) + }, f) diff --git a/sdk/cwl/tests/federation/framework/testcase.cwl b/sdk/cwl/tests/federation/framework/testcase.cwl new file mode 100644 index 0000000000..89aa3f98b9 --- /dev/null +++ b/sdk/cwl/tests/federation/framework/testcase.cwl @@ -0,0 +1,77 @@ +#!/usr/bin/env cwl-runner +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +hints: + cwltool:Secrets: + secrets: [arvados_api_token] +requirements: + StepInputExpressionRequirement: {} + InlineJavascriptRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + arvados_api_token: string + arvado_api_host_insecure: + type: boolean + default: false + arvados_api_hosts: string[] + arvados_cluster_ids: string[] + acr: string? + wf: File + obj: Any + scrub_image: string + scrub_collections: string[] + runner_cluster: string? +outputs: + out: + type: Any + outputSource: run-acr/out + success: + type: boolean + outputSource: check-result/success +steps: + dockerbuild: + in: + testcase: scrub_image + out: [imagename] + run: dockerbuild.cwl + prepare: + in: + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"} + arvados_cluster_ids: arvados_cluster_ids + wf: wf + obj: obj + scrub_image: scrub_image + scrub_collections: scrub_collections + out: [done] + run: prepare.cwl + run-acr: + in: + prepare: prepare/done + image-ready: dockerbuild/imagename + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"} + runner_cluster: runner_cluster + acr: acr + wf: wf + obj: obj + out: [out] + run: run-acr.cwl + check-result: + in: + acr-done: run-acr/out + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_host: {source: arvados_api_hosts, valueFrom: "$(self[0])"} + check_collections: scrub_collections + out: [success] + run: check-exist.cwl \ No newline at end of file diff --git a/sdk/cwl/tests/federation/main.cwl b/sdk/cwl/tests/federation/main.cwl new file mode 100755 index 0000000000..1042350f93 --- /dev/null +++ b/sdk/cwl/tests/federation/main.cwl @@ -0,0 +1,143 @@ +#!/usr/bin/env cwl-runner +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +cwlVersion: v1.0 +class: Workflow +$namespaces: + arv: "http://arvados.org/cwl#" + cwltool: "http://commonwl.org/cwltool#" +hints: + cwltool:Secrets: + secrets: [arvados_api_token] +requirements: + StepInputExpressionRequirement: {} + InlineJavascriptRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + arvados_api_token: string + arvado_api_host_insecure: + type: boolean + default: false + arvados_api_hosts: string[] + arvados_cluster_ids: string[] + acr: string? + testcases: + type: string[] + default: + - base-case + - runner-home-step-remote + - runner-remote-step-home +outputs: + base-case-success: + type: Any + outputSource: base-case/success + runner-home-step-remote-success: + type: Any + outputSource: runner-home-step-remote/success + runner-remote-step-home-success: + type: Any + outputSource: runner-remote-step-home/success + +steps: + base-case: + in: + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_hosts: arvados_api_hosts + arvados_cluster_ids: arvados_cluster_ids + acr: acr + wf: + default: + class: File + location: cases/base-case.cwl + secondaryFiles: + - class: File + location: cases/md5sum.cwl + obj: + default: + inp: + class: File + location: data/base-case-input.txt + valueFrom: |- + ${ + self["runOnCluster"] = inputs.arvados_cluster_ids[0]; + return self; + } + scrub_image: {default: "arvados/fed-test:base-case"} + scrub_collections: + default: + - 031a4ced0aa99de90fb630568afc6e9b+67 # input collection + - eb93a6718eb1a1a8ee9f66ee7d683472+51 # md5sum output collection + - f654d4048612135f4a5e7707ec0fcf3e+112 # final output json + out: [out, success] + run: framework/testcase.cwl + + runner-home-step-remote: + in: + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_hosts: arvados_api_hosts + arvados_cluster_ids: arvados_cluster_ids + acr: acr + wf: + default: + class: File + location: cases/runner-home-step-remote.cwl + secondaryFiles: + - class: File + location: cases/md5sum.cwl + obj: + default: + inp: + class: File + location: data/runner-home-step-remote-input.txt + valueFrom: |- + ${ + self["runOnCluster"] = inputs.arvados_cluster_ids[1]; + return self; + } + runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" } + scrub_image: {default: "arvados/fed-test:runner-home-step-remote"} + scrub_collections: + default: + - 3bc373e38751fe13dcbd62778d583242+81 # input collection + - 428e6d91e41a3af3ae287b453949e7fd+51 # md5sum output collection + - a4b0ddd866525655e8480f83a1ca83c6+112 # runner output json + out: [out, success] + run: framework/testcase.cwl + + runner-remote-step-home: + in: + arvados_api_token: arvados_api_token + arvado_api_host_insecure: arvado_api_host_insecure + arvados_api_hosts: arvados_api_hosts + arvados_cluster_ids: arvados_cluster_ids + acr: acr + wf: + default: + class: File + location: cases/runner-remote-step-home.cwl + secondaryFiles: + - class: File + location: cases/md5sum.cwl + obj: + default: + inp: + class: File + location: data/runner-remote-step-home-input.txt + valueFrom: |- + ${ + self["runOnCluster"] = inputs.arvados_cluster_ids[0]; + return self; + } + runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" } + scrub_image: {default: "arvados/fed-test:runner-remote-step-home"} + scrub_collections: + default: + - 25fe10d8e8530329a738de69d9bc8ab5+81 # input collection + - 7f052d1a04b851b6f73fba77c7802e1d+51 # md5sum output collection + - ecb639201f454b6493757f5117f540df+112 # runner output json + out: [out, success] + run: framework/testcase.cwl -- 2.30.2