sdk/cwl/tests/test_copy_deps.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import arvados.collection
   7 import subprocess
   8 import json
   9
  10 api = arvados.api()
  11
  12 workflow_content = """{
  13     "$graph": [
  14         {
  15             "baseCommand": "echo",
  16             "class": "CommandLineTool",
  17             "cwlVersion": "v1.2",
  18             "hints": [
  19                 {
  20                     "class": "http://arvados.org/cwl#WorkflowRunnerResources"
  21                 }
  22             ],
  23             "id": "#main",
  24             "inputs": [
  25                 {
  26                     "default": {
  27                         "basename": "b",
  28                         "class": "File",
  29                         "location": "keep:d7514270f356df848477718d58308cc4+94/b",
  30                         "nameext": "",
  31                         "nameroot": "b",
  32                         "size": 0
  33                     },
  34                     "id": "#main/message",
  35                     "inputBinding": {
  36                         "position": 1
  37                     },
  38                     "type": "File"
  39                 }
  40             ],
  41             "outputs": []
  42         }
  43     ],
  44     "cwlVersion": "v1.2"
  45 }"""
  46
  47 def check_workflow_content(uuid):
  48     c = arvados.collection.Collection(uuid)
  49     try:
  50         j = json.load(c.open("workflow.json"))
  51     except IOError:
  52         return False
  53     # The value of "acrContainerImage" is tied to the specific version
  54     # of arvados-cwl-runner so we can't just compare PDH of the whole
  55     # workflow collection, it changes with every version.
  56     del j["$graph"][0]["hints"][0]["acrContainerImage"]
  57     print
  58     return json.dumps(j, sort_keys=True, indent=4, separators=(',',': ')) == workflow_content
  59
  60 def check_contents(group, wf_uuid):
  61     contents = api.groups().contents(uuid=group["uuid"]).execute()
  62     if len(contents["items"]) != 4:
  63         raise Exception("Expected 4 items in "+group["uuid"]+" was "+str(len(contents["items"])))
  64
  65     found = False
  66     for c in contents["items"]:
  67         if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  68             found = True
  69     if not found:
  70         raise Exception("Couldn't find workflow in "+group["uuid"])
  71
  72     found = False
  73     for c in contents["items"]:
  74         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
  75             found = True
  76     if not found:
  77         raise Exception("Couldn't find collection dependency")
  78
  79     found = False
  80     for c in contents["items"]:
  81         if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
  82             found = True
  83     if not found:
  84         raise Exception("Couldn't find jobs image dependency")
  85
  86     found = False
  87     for c in contents["items"]:
  88         if c["kind"] == "arvados#collection" and check_workflow_content(c["portable_data_hash"]):
  89             found = True
  90     if not found:
  91         raise Exception("Couldn't find collection containing expected workflow.json")
  92
  93
  94 def test_create():
  95     group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
  96     try:
  97         contents = api.groups().contents(uuid=group["uuid"]).execute()
  98         if len(contents["items"]) != 0:
  99             raise Exception("Expected 0 items")
 100
 101         # Create workflow, by default should also copy dependencies
 102         cmd = ["arvados-cwl-runner", "--disable-git", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 103         print(" ".join(cmd))
 104         wf_uuid = subprocess.check_output(cmd)
 105         wf_uuid = wf_uuid.decode("utf-8").strip()
 106         check_contents(group, wf_uuid)
 107     finally:
 108         api.groups().delete(uuid=group["uuid"]).execute()
 109
 110
 111 def test_update():
 112     group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
 113     try:
 114         contents = api.groups().contents(uuid=group["uuid"]).execute()
 115         if len(contents["items"]) != 0:
 116             raise Exception("Expected 0 items")
 117
 118         # Create workflow, but with --no-copy-deps it shouldn't copy anything
 119         cmd = ["arvados-cwl-runner", "--disable-git", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 120         print(" ".join(cmd))
 121         wf_uuid = subprocess.check_output(cmd)
 122         wf_uuid = wf_uuid.decode("utf-8").strip()
 123
 124         contents = api.groups().contents(uuid=group["uuid"]).execute()
 125         if len(contents["items"]) != 2:
 126             raise Exception("Expected 2 items")
 127
 128         found = False
 129         for c in contents["items"]:
 130             if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
 131                 found = True
 132         if not found:
 133             raise Exception("Couldn't find workflow")
 134
 135         found = False
 136         for c in contents["items"]:
 137             if c["kind"] == "arvados#collection" and check_workflow_content(c["portable_data_hash"]):
 138                 found = True
 139         if not found:
 140             raise Exception("Couldn't find collection containing expected workflow.json")
 141
 142         # Updating by default will copy missing items
 143         cmd = ["arvados-cwl-runner", "--disable-git", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]
 144         print(" ".join(cmd))
 145         wf_uuid = subprocess.check_output(cmd)
 146         wf_uuid = wf_uuid.decode("utf-8").strip()
 147         check_contents(group, wf_uuid)
 148
 149     finally:
 150         api.groups().delete(uuid=group["uuid"]).execute()
 151
 152
 153 def test_execute():
 154     group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
 155     try:
 156         contents = api.groups().contents(uuid=group["uuid"]).execute()
 157         if len(contents["items"]) != 0:
 158             raise Exception("Expected 0 items")
 159
 160         # Execute workflow, shouldn't copy anything.
 161         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
 162         print(" ".join(cmd))
 163         wf_uuid = subprocess.check_output(cmd)
 164         wf_uuid = wf_uuid.decode("utf-8").strip()
 165
 166         contents = api.groups().contents(uuid=group["uuid"]).execute()
 167         # container request
 168         # final output collection
 169         # container log
 170         # step output collection
 171         # container request log
 172         if len(contents["items"]) != 5:
 173             raise Exception("Expected 5 items")
 174
 175         found = False
 176         for c in contents["items"]:
 177             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 178                 found = True
 179         if found:
 180             raise Exception("Didn't expect to find collection dependency")
 181
 182         found = False
 183         for c in contents["items"]:
 184             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 185                 found = True
 186         if found:
 187             raise Exception("Didn't expect to find jobs image dependency")
 188
 189         # Execute workflow with --copy-deps
 190         cmd = ["arvados-cwl-runner", "--disable-git", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]
 191         print(" ".join(cmd))
 192         wf_uuid = subprocess.check_output(cmd)
 193         wf_uuid = wf_uuid.decode("utf-8").strip()
 194
 195         contents = api.groups().contents(uuid=group["uuid"]).execute()
 196         found = False
 197         for c in contents["items"]:
 198             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 199                 found = True
 200         if not found:
 201             raise Exception("Couldn't find collection dependency")
 202
 203         found = False
 204         for c in contents["items"]:
 205             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 206                 found = True
 207         if not found:
 208             raise Exception("Couldn't find jobs image dependency")
 209
 210     finally:
 211         api.groups().delete(uuid=group["uuid"]).execute()
 212
 213 if __name__ == '__main__':
 214     test_create()
 215     test_update()
 216     test_execute()