sdk/cwl/tests/test_copy_deps.py

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4
   5 import arvados
   6 import subprocess
   7
   8 api = arvados.api()
   9
  10 def check_contents(group, wf_uuid):
  11     contents = api.groups().contents(uuid=group["uuid"]).execute()
  12     if len(contents["items"]) != 3:
  13         raise Exception("Expected 3 items")
  14
  15     found = False
  16     for c in contents["items"]:
  17         if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  18             found = True
  19     if not found:
  20         raise Exception("Couldn't find workflow")
  21
  22     found = False
  23     for c in contents["items"]:
  24         if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
  25             found = True
  26     if not found:
  27         raise Exception("Couldn't find collection dependency")
  28
  29     found = False
  30     for c in contents["items"]:
  31         if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
  32             found = True
  33     if not found:
  34         raise Exception("Couldn't find jobs image dependency")
  35
  36
  37 def test_create():
  38     group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
  39     try:
  40         contents = api.groups().contents(uuid=group["uuid"]).execute()
  41         if len(contents["items"]) != 0:
  42             raise Exception("Expected 0 items")
  43
  44         # Create workflow, by default should also copy dependencies
  45         wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
  46         wf_uuid = wf_uuid.decode("utf-8").strip()
  47         check_contents(group, wf_uuid)
  48     finally:
  49         api.groups().delete(uuid=group["uuid"]).execute()
  50
  51
  52 def test_update():
  53     group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
  54     try:
  55         contents = api.groups().contents(uuid=group["uuid"]).execute()
  56         if len(contents["items"]) != 0:
  57             raise Exception("Expected 0 items")
  58
  59         # Create workflow, but with --no-copy-deps it shouldn't copy anything
  60         wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--no-copy-deps", "--create-workflow", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
  61         wf_uuid = wf_uuid.decode("utf-8").strip()
  62
  63         contents = api.groups().contents(uuid=group["uuid"]).execute()
  64         if len(contents["items"]) != 1:
  65             raise Exception("Expected 1 items")
  66
  67         found = False
  68         for c in contents["items"]:
  69             if c["kind"] == "arvados#workflow" and c["uuid"] == wf_uuid:
  70                 found = True
  71         if not found:
  72             raise Exception("Couldn't find workflow")
  73
  74         # Updating by default will copy missing items
  75         wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--update-workflow", wf_uuid, "19070-copy-deps.cwl"])
  76         wf_uuid = wf_uuid.decode("utf-8").strip()
  77         check_contents(group, wf_uuid)
  78
  79     finally:
  80         api.groups().delete(uuid=group["uuid"]).execute()
  81
  82
  83 def test_execute():
  84     group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
  85     try:
  86         contents = api.groups().contents(uuid=group["uuid"]).execute()
  87         if len(contents["items"]) != 0:
  88             raise Exception("Expected 0 items")
  89
  90         # Execute workflow, shouldn't copy anything.
  91         wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "19070-copy-deps.cwl"])
  92         wf_uuid = wf_uuid.decode("utf-8").strip()
  93
  94         contents = api.groups().contents(uuid=group["uuid"]).execute()
  95         # container request
  96         # final output collection
  97         # container log
  98         # step output collection
  99         # container request log
 100         if len(contents["items"]) != 5:
 101             raise Exception("Expected 5 items")
 102
 103         found = False
 104         for c in contents["items"]:
 105             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 106                 found = True
 107         if found:
 108             raise Exception("Didn't expect to find collection dependency")
 109
 110         found = False
 111         for c in contents["items"]:
 112             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 113                 found = True
 114         if found:
 115             raise Exception("Didn't expect to find jobs image dependency")
 116
 117         # Execute workflow with --copy-deps
 118         wf_uuid = subprocess.check_output(["arvados-cwl-runner", "--project-uuid", group["uuid"], "--copy-deps", "19070-copy-deps.cwl"])
 119         wf_uuid = wf_uuid.decode("utf-8").strip()
 120
 121         contents = api.groups().contents(uuid=group["uuid"]).execute()
 122         found = False
 123         for c in contents["items"]:
 124             if c["kind"] == "arvados#collection" and c["portable_data_hash"] == "d7514270f356df848477718d58308cc4+94":
 125                 found = True
 126         if not found:
 127             raise Exception("Couldn't find collection dependency")
 128
 129         found = False
 130         for c in contents["items"]:
 131             if c["kind"] == "arvados#collection" and c["name"].startswith("Docker image arvados jobs"):
 132                 found = True
 133         if not found:
 134             raise Exception("Couldn't find jobs image dependency")
 135
 136     finally:
 137         api.groups().delete(uuid=group["uuid"]).execute()
 138
 139 if __name__ == '__main__':
 140     test_create()
 141     test_update()
 142     test_execute()