From 341f5b9ee96921018d80145712bba1200f9ea1a3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 12 Mar 2018 14:37:56 -0400 Subject: [PATCH] 13135: Update tests for secret_mounts Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/cwl/tests/test_container.py | 115 ++++++++++++++++++++++- sdk/cwl/tests/test_submit.py | 159 +++++++++++++++++++++++++++++++- sdk/cwl/tests/wf/secret_job.cwl | 2 +- 3 files changed, 268 insertions(+), 8 deletions(-) diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py index cd555a72ca..522946a4f4 100644 --- a/sdk/cwl/tests/test_container.py +++ b/sdk/cwl/tests/test_container.py @@ -10,6 +10,7 @@ import unittest import os import functools import cwltool.process +import cwltool.secrets from schema_salad.ref_resolver import Loader from schema_salad.sourceline import cmap @@ -33,6 +34,7 @@ class TestContainer(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -85,6 +87,7 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, + 'secret_mounts': {} })) # The test passes some fields in builder.resources @@ -96,6 +99,8 @@ class TestContainer(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 3600 + runner.secret_store = cwltool.secrets.SecretStore() + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] @@ -172,7 +177,8 @@ class TestContainer(unittest.TestCase): 'scheduling_parameters': { 'partitions': ['blurb'] }, - 'properties': {} + 'properties': {}, + 'secret_mounts': {} } call_body = call_kwargs.get('body', None) @@ -191,6 +197,8 @@ class TestContainer(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] @@ -303,7 +311,8 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': { }, - 'properties': {} + 'properties': {}, + 'secret_mounts': {} } call_body = call_kwargs.get('body', None) @@ -321,6 +330,7 @@ class TestContainer(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -388,6 +398,7 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, + 'secret_mounts': {} })) @mock.patch("arvados.collection.Collection") @@ -400,6 +411,7 @@ class TestContainer(unittest.TestCase): runner.num_retries = 0 runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() runner.api.containers().get().execute.return_value = {"state":"Complete", "output": "abc+123", @@ -443,6 +455,7 @@ class TestContainer(unittest.TestCase): runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" runner.ignore_docker_for_reuse = False runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] runner.api.collections().get().execute.return_value = { @@ -517,4 +530,102 @@ class TestContainer(unittest.TestCase): 'cwd': '/var/spool/cwl', 'scheduling_parameters': {}, 'properties': {}, + 'secret_mounts': {} + })) + + # The test passes no builder.resources + # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024} + @mock.patch("arvados.commands.keepdocker.list_images_in_arv") + def test_secrets(self, keepdocker): + arv_docker_clear_cache() + + runner = mock.MagicMock() + runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz" + runner.ignore_docker_for_reuse = False + runner.intermediate_output_ttl = 0 + runner.secret_store = cwltool.secrets.SecretStore() + + keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")] + runner.api.collections().get().execute.return_value = { + "portable_data_hash": "99999999999999999999999999999993+99"} + + document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0") + + tool = cmap({"arguments": ["md5sum", "example.conf"], + "class": "CommandLineTool", + "hints": [ + { + "class": "http://commonwl.org/cwltool#Secrets", + "secrets": [ + "#secret_job.cwl/pw" + ] + } + ], + "id": "#secret_job.cwl", + "inputs": [ + { + "id": "#secret_job.cwl/pw", + "type": "string" + } + ], + "outputs": [ + ], + "requirements": [ + { + "class": "InitialWorkDirRequirement", + "listing": [ + { + "entry": "username: user\npassword: $(inputs.pw)\n", + "entryname": "example.conf" + } + ] + } + ]}) + make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess, + collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0)) + arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, work_api="containers", avsc_names=avsc_names, + basedir="", make_fs_access=make_fs_access, loader=Loader({})) + arvtool.formatgraph = None + + job_order = {"pw": "blorp"} + runner.secret_store.store(["pw"], job_order) + + for j in arvtool.job(job_order, mock.MagicMock(), basedir="", name="test_secrets", + make_fs_access=make_fs_access, tmpdir="/tmp"): + j.run(enable_reuse=True, priority=500) + runner.api.container_requests().create.assert_called_with( + body=JsonDiffMatcher({ + 'environment': { + 'HOME': '/var/spool/cwl', + 'TMPDIR': '/tmp' + }, + 'name': 'test_secrets', + 'runtime_constraints': { + 'vcpus': 1, + 'ram': 1073741824 + }, + 'use_existing': True, + 'priority': 500, + 'mounts': { + '/tmp': {'kind': 'tmp', + "capacity": 1073741824 + }, + '/var/spool/cwl': {'kind': 'tmp', + "capacity": 1073741824 } + }, + 'state': 'Committed', + 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz', + 'output_path': '/var/spool/cwl', + 'output_ttl': 0, + 'container_image': 'arvados/jobs', + 'command': ['md5sum', 'example.conf'], + 'cwd': '/var/spool/cwl', + 'scheduling_parameters': {}, + 'properties': {}, + "secret_mounts": { + "/var/spool/cwl/example.conf": { + "content": "username: user\npassword: blorp\n", + "kind": "text" + } + } })) diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py index c7c94fd7b9..5ab452fdfc 100644 --- a/sdk/cwl/tests/test_submit.py +++ b/sdk/cwl/tests/test_submit.py @@ -247,7 +247,8 @@ def stubs(func): 'ram': 1024*1024*1024 }, 'use_existing': True, - 'properties': {} + 'properties': {}, + 'secret_mounts': {} } stubs.expect_workflow_uuid = "zzzzz-7fd4e-zzzzzzzzzzzzzzz" @@ -745,7 +746,8 @@ class TestSubmit(unittest.TestCase): 'ram': 1073741824 }, 'use_existing': True, - 'properties': {} + 'properties': {}, + 'secret_mounts': {} } stubs.api.container_requests().create.assert_called_with( @@ -863,7 +865,8 @@ class TestSubmit(unittest.TestCase): 'use_existing': True, 'properties': { "template_uuid": "962eh-7fd4e-gkbzl62qqtfig37" - } + }, + 'secret_mounts': {} } stubs.api.container_requests().create.assert_called_with( @@ -1040,13 +1043,159 @@ class TestSubmit(unittest.TestCase): try: exited = arvados_cwl.main( ["--submit", "--no-wait", "--api=containers", "--debug", - "tests/wf/secret_wf.cwl", "tests/submit_test_job.json"], + "tests/wf/secret_wf.cwl", "tests/secret_test_job.yml"], capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client) self.assertEqual(exited, 0) except: logging.exception("") - expect_container = copy.deepcopy(stubs.expect_container_spec) + + expect_container = { + "command": [ + "arvados-cwl-runner", + "--local", + "--api=containers", + "--no-log-timestamps", + "--enable-reuse", + "--on-error=continue", + "--eval-timeout=20", + "/var/lib/cwl/workflow.json#main", + "/var/lib/cwl/cwl.input.json" + ], + "container_image": "arvados/jobs:"+arvados_cwl.__version__, + "cwd": "/var/spool/cwl", + "mounts": { + "/var/lib/cwl/cwl.input.json": { + "content": { + "pw": { + "$include": "/secrets/s0" + } + }, + "kind": "json" + }, + "/var/lib/cwl/workflow.json": { + "content": { + "$graph": [ + { + "$namespaces": { + "cwltool": "http://commonwl.org/cwltool#" + }, + "arguments": [ + "md5sum", + "example.conf" + ], + "class": "CommandLineTool", + "hints": [ + { + "class": "http://commonwl.org/cwltool#Secrets", + "secrets": [ + "#secret_job.cwl/pw" + ] + } + ], + "id": "#secret_job.cwl", + "inputs": [ + { + "id": "#secret_job.cwl/pw", + "type": "string" + } + ], + "outputs": [ + { + "id": "#secret_job.cwl/out", + "type": "stdout" + } + ], + "requirements": [ + { + "class": "InitialWorkDirRequirement", + "listing": [ + { + "entry": "username: user\npassword: $(inputs.pw)\n", + "entryname": "example.conf" + } + ] + } + ] + }, + { + "class": "Workflow", + "hints": [ + { + "class": "DockerRequirement", + "dockerPull": "debian:8" + }, + { + "class": "http://commonwl.org/cwltool#Secrets", + "secrets": [ + "#main/pw" + ] + } + ], + "id": "#main", + "inputs": [ + { + "id": "#main/pw", + "type": "string" + } + ], + "outputs": [ + { + "id": "#main/out", + "outputSource": "#main/step1/out", + "type": "File" + } + ], + "steps": [ + { + "id": "#main/step1", + "in": [ + { + "id": "#main/step1/pw", + "source": "#main/pw" + } + ], + "out": [ + "#main/step1/out" + ], + "run": "#secret_job.cwl" + } + ] + } + ], + "cwlVersion": "v1.0" + }, + "kind": "json" + }, + "/var/spool/cwl": { + "kind": "collection", + "writable": True + }, + "stdout": { + "kind": "file", + "path": "/var/spool/cwl/cwl.output.json" + } + }, + "name": "secret_wf.cwl", + "output_path": "/var/spool/cwl", + "owner_uuid": None, + "priority": 500, + "properties": {}, + "runtime_constraints": { + "API": True, + "ram": 1073741824, + "vcpus": 1 + }, + "secret_mounts": { + "/secrets/s0": { + "content": "blorp", + "kind": "text" + } + }, + "state": "Committed", + "use_existing": True + } + stubs.api.container_requests().create.assert_called_with( body=JsonDiffMatcher(expect_container)) self.assertEqual(capture_stdout.getvalue(), diff --git a/sdk/cwl/tests/wf/secret_job.cwl b/sdk/cwl/tests/wf/secret_job.cwl index 40e18e1304..aa68905d9c 100644 --- a/sdk/cwl/tests/wf/secret_job.cwl +++ b/sdk/cwl/tests/wf/secret_job.cwl @@ -16,4 +16,4 @@ inputs: pw: string outputs: out: stdout -arguments: [cat, example.conf] +arguments: [md5sum, example.conf] -- 2.30.2