sdk/cwl/tests/test_copy_deps.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import subprocess
   7
   8 api = arvados.api()
   9
  10 def check_contents(group, wf_uuid):
  11     contents = api.groups().contents(uuid=group["uuid"]).execute()
  12     if len(contents["items"]) != 3:
  13         raise Exception("Expected 3 items in "+group["uuid"]+" was "+len(contents["items"]))
  14
  15     found = False
  16     for c in contents["items"]:
  17         if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  18             found = True
  19     if not found:
  20         raise Exception("Couldn't find workflow in "+group["uuid"])
  21
  22     found = False
  23     for c in contents["items"]:
  24         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
  25             found = True
  26     if not found:
  27         raise Exception("Couldn't find collection dependency")
  28
  29     found = False
  30     for c in contents["items"]:
  31         if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
  32             found = True
  33     if not found:
  34         raise Exception("Couldn't find jobs image dependency")
  35
  36
  37 def test_create():
  38     group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
  39     try:
  40         contents = api.groups().contents(uuid=group["uuid"]).execute()
  41         if len(contents["items"]) != 0:
  42             raise Exception("Expected 0 items")
  43
  44         # Create workflow, by default should also copy dependencies
  45         cmd = ["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  46         print(" ".join(cmd))
  47         wf_uuid = subprocess.check_output(cmd)
  48         wf_uuid = wf_uuid.decode("utf-8").strip()
  49         check_contents(group, wf_uuid)
  50     finally:
  51         api.groups().delete(uuid=group["uuid"]).execute()
  52
  53
  54 def test_update():
  55     group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
  56     try:
  57         contents = api.groups().contents(uuid=group["uuid"]).execute()
  58         if len(contents["items"]) != 0:
  59             raise Exception("Expected 0 items")
  60
  61         # Create workflow, but with --no-copy-deps it shouldn't copy anything
  62         cmd = ["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  63         print(" ".join(cmd))
  64         wf_uuid = subprocess.check_output(cmd)
  65         wf_uuid = wf_uuid.decode("utf-8").strip()
  66
  67         contents = api.groups().contents(uuid=group["uuid"]).execute()
  68         if len(contents["items"]) != 1:
  69             raise Exception("Expected 1 items")
  70
  71         found = False
  72         for c in contents["items"]:
  73             if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  74                 found = True
  75         if not found:
  76             raise Exception("Couldn't find workflow")
  77
  78         # Updating by default will copy missing items
  79         cmd = ["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"]
  80         print(" ".join(cmd))
  81         wf_uuid = subprocess.check_output(cmd)
  82         wf_uuid = wf_uuid.decode("utf-8").strip()
  83         check_contents(group, wf_uuid)
  84
  85     finally:
  86         api.groups().delete(uuid=group["uuid"]).execute()
  87
  88
  89 def test_execute():
  90     group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
  91     try:
  92         contents = api.groups().contents(uuid=group["uuid"]).execute()
  93         if len(contents["items"]) != 0:
  94             raise Exception("Expected 0 items")
  95
  96         # Execute workflow, shouldn't copy anything.
  97         cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"]
  98         print(" ".join(cmd))
  99         wf_uuid = subprocess.check_output(cmd)
 100         wf_uuid = wf_uuid.decode("utf-8").strip()
 101
 102         contents = api.groups().contents(uuid=group["uuid"]).execute()
 103         # container request
 104         # final output collection
 105         # container log
 106         # step output collection
 107         # container request log
 108         if len(contents["items"]) != 5:
 109             raise Exception("Expected 5 items")
 110
 111         found = False
 112         for c in contents["items"]:
 113             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 114                 found = True
 115         if found:
 116             raise Exception("Didn't expect to find collection dependency")
 117
 118         found = False
 119         for c in contents["items"]:
 120             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 121                 found = True
 122         if found:
 123             raise Exception("Didn't expect to find jobs image dependency")
 124
 125         # Execute workflow with --copy-deps
 126         cmd = ["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"]
 127         print(" ".join(cmd))
 128         wf_uuid = subprocess.check_output(cmd)
 129         wf_uuid = wf_uuid.decode("utf-8").strip()
 130
 131         contents = api.groups().contents(uuid=group["uuid"]).execute()
 132         found = False
 133         for c in contents["items"]:
 134             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 135                 found = True
 136         if not found:
 137             raise Exception("Couldn't find collection dependency")
 138
 139         found = False
 140         for c in contents["items"]:
 141             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 142                 found = True
 143         if not found:
 144             raise Exception("Couldn't find jobs image dependency")
 145
 146     finally:
 147         api.groups().delete(uuid=group["uuid"]).execute()
 148
 149 if __name__ == '__main__':
 150     test_create()
 151     test_update()
 152     test_execute()