sdk/cwl/tests/federation/main.cwl

   1 #!/usr/bin/env cwl-runner
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: Apache-2.0
   5
   6 cwlVersion: v1.0
   7 class: Workflow
   8 $namespaces:
   9   arv: "http://arvados.org/cwl#"
  10   cwltool: "http://commonwl.org/cwltool#"
  11 hints:
  12   cwltool:Secrets:
  13     secrets: [arvados_api_token]
  14 requirements:
  15   StepInputExpressionRequirement: {}
  16   InlineJavascriptRequirement: {}
  17   SubworkflowFeatureRequirement: {}
  18 inputs:
  19   arvados_api_token: string
  20   arvado_api_host_insecure:
  21     type: boolean
  22     default: false
  23   arvados_api_hosts: string[]
  24   arvados_cluster_ids: string[]
  25   acr: string?
  26   testcases:
  27     type: string[]
  28     default:
  29       - base-case
  30       - runner-home-step-remote
  31       - runner-remote-step-home
  32 outputs:
  33   base-case-success:
  34     type: Any
  35     outputSource: base-case/success
  36   runner-home-step-remote-success:
  37     type: Any
  38     outputSource: runner-home-step-remote/success
  39   runner-remote-step-home-success:
  40     type: Any
  41     outputSource: runner-remote-step-home/success
  42   remote-case-success:
  43     type: Any
  44     outputSource: remote-case/success
  45   twostep-home-to-remote-success:
  46     type: Any
  47     outputSource: twostep-home-to-remote/success
  48   twostep-remote-to-home-success:
  49     type: Any
  50     outputSource: twostep-remote-to-home/success
  51   twostep-both-remote-success:
  52     type: Any
  53     outputSource: twostep-both-remote/success
  54   twostep-remote-copy-to-home-success:
  55     type: Any
  56     outputSource: twostep-remote-copy-to-home/success
  57   scatter-gather-success:
  58     type: Any
  59     outputSource: scatter-gather/success
  60   threestep-remote-success:
  61     type: Any
  62     outputSource: threestep-remote/success
  63   hint-on-wf-success:
  64     type: Any
  65     outputSource: hint-on-wf/success
  66   hint-on-tool-success:
  67     type: Any
  68     outputSource: hint-on-tool/success
  69
  70 steps:
  71   base-case:
  72     doc: |
  73       Base case (no federation), single step workflow with both the
  74       runner and step on the same cluster.
  75     in:
  76       arvados_api_token: arvados_api_token
  77       arvado_api_host_insecure: arvado_api_host_insecure
  78       arvados_api_hosts: arvados_api_hosts
  79       arvados_cluster_ids: arvados_cluster_ids
  80       acr: acr
  81       wf:
  82         default:
  83           class: File
  84           location: cases/base-case.cwl
  85           secondaryFiles:
  86             - class: File
  87               location: cases/md5sum.cwl
  88       obj:
  89         default:
  90           inp:
  91             class: File
  92             location: data/base-case-input.txt
  93         valueFrom: |-
  94           ${
  95           self["runOnCluster"] = inputs.arvados_cluster_ids[0];
  96           return self;
  97           }
  98       scrub_image: {default: "arvados/fed-test:base-case"}
  99       scrub_collections:
 100         default:
 101           - 031a4ced0aa99de90fb630568afc6e9b+67   # input collection
 102           - eb93a6718eb1a1a8ee9f66ee7d683472+51   # md5sum output collection
 103           - f654d4048612135f4a5e7707ec0fcf3e+112  # final output json
 104     out: [out, success]
 105     run: framework/testcase.cwl
 106
 107   runner-home-step-remote:
 108     doc: |
 109       Single step workflow with the runner on the home cluster and the
 110       step on the remote cluster.  ClusterTarget hint is on the workflow step.
 111     in:
 112       arvados_api_token: arvados_api_token
 113       arvado_api_host_insecure: arvado_api_host_insecure
 114       arvados_api_hosts: arvados_api_hosts
 115       arvados_cluster_ids: arvados_cluster_ids
 116       acr: acr
 117       wf:
 118         default:
 119           class: File
 120           location: cases/runner-home-step-remote.cwl
 121           secondaryFiles:
 122             - class: File
 123               location: cases/md5sum.cwl
 124       obj:
 125         default:
 126           inp:
 127             class: File
 128             location: data/runner-home-step-remote-input.txt
 129         valueFrom: |-
 130           ${
 131           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 132           return self;
 133           }
 134       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 135       scrub_image: {default: "arvados/fed-test:runner-home-step-remote"}
 136       scrub_collections:
 137         default:
 138           - 3bc373e38751fe13dcbd62778d583242+81   # input collection
 139           - 428e6d91e41a3af3ae287b453949e7fd+51   # md5sum output collection
 140           - a4b0ddd866525655e8480f83a1ca83c6+112  # runner output json
 141     out: [out, success]
 142     run: framework/testcase.cwl
 143
 144   runner-remote-step-home:
 145     doc: |
 146       Single step workflow with the runner on the remote cluster and the
 147       step on the home cluster.
 148     in:
 149       arvados_api_token: arvados_api_token
 150       arvado_api_host_insecure: arvado_api_host_insecure
 151       arvados_api_hosts: arvados_api_hosts
 152       arvados_cluster_ids: arvados_cluster_ids
 153       acr: acr
 154       wf:
 155         default:
 156           class: File
 157           location: cases/runner-remote-step-home.cwl
 158           secondaryFiles:
 159             - class: File
 160               location: cases/md5sum.cwl
 161       obj:
 162         default:
 163           inp:
 164             class: File
 165             location: data/runner-remote-step-home-input.txt
 166         valueFrom: |-
 167           ${
 168           self["runOnCluster"] = inputs.arvados_cluster_ids[0];
 169           return self;
 170           }
 171       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
 172       scrub_image: {default: "arvados/fed-test:runner-remote-step-home"}
 173       scrub_collections:
 174         default:
 175           - 25fe10d8e8530329a738de69d9bc8ab5+81   # input collection
 176           - 7f052d1a04b851b6f73fba77c7802e1d+51   # md5sum output collection
 177           - ecb639201f454b6493757f5117f540df+112  # runner output json
 178     out: [out, success]
 179     run: framework/testcase.cwl
 180
 181   remote-case:
 182     doc: |
 183       Single step workflow with both the runner and the step on the
 184       remote cluster.
 185     in:
 186       arvados_api_token: arvados_api_token
 187       arvado_api_host_insecure: arvado_api_host_insecure
 188       arvados_api_hosts: arvados_api_hosts
 189       arvados_cluster_ids: arvados_cluster_ids
 190       acr: acr
 191       wf:
 192         default:
 193           class: File
 194           location: cases/remote-case.cwl
 195           secondaryFiles:
 196             - class: File
 197               location: cases/md5sum.cwl
 198       obj:
 199         default:
 200           inp:
 201             class: File
 202             location: data/remote-case-input.txt
 203         valueFrom: |-
 204           ${
 205           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 206           return self;
 207           }
 208       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[1])" }
 209       scrub_image: {default: "arvados/fed-test:remote-case"}
 210       scrub_collections:
 211         default:
 212           - fccd49fdef8e452295f718208abafd88+69   # input collection
 213           - 58c0e8ea6b148134ef8577ee11307eec+51   # md5sum output collection
 214           - 1fd679c5ab64c123b9764024dbf560f0+112  # final output json
 215     out: [out, success]
 216     run: framework/testcase.cwl
 217
 218   twostep-home-to-remote:
 219     doc: |
 220       Two step workflow.  The runner is on the home cluster, the first
 221       step is on the home cluster, the second step is on the remote
 222       cluster.
 223     in:
 224       arvados_api_token: arvados_api_token
 225       arvado_api_host_insecure: arvado_api_host_insecure
 226       arvados_api_hosts: arvados_api_hosts
 227       arvados_cluster_ids: arvados_cluster_ids
 228       acr: acr
 229       wf:
 230         default:
 231           class: File
 232           location: cases/twostep-home-to-remote.cwl
 233           secondaryFiles:
 234             - class: File
 235               location: cases/md5sum.cwl
 236             - class: File
 237               location: cases/rev.cwl
 238       obj:
 239         default:
 240           inp:
 241             class: File
 242             location: data/twostep-home-to-remote.txt
 243         valueFrom: |-
 244           ${
 245           self["md5sumCluster"] = inputs.arvados_cluster_ids[0];
 246           self["revCluster"] = inputs.arvados_cluster_ids[1];
 247           return self;
 248           }
 249       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 250       scrub_image: {default: "arvados/fed-test:twostep-home-to-remote"}
 251       scrub_collections:
 252         default:
 253           - 268a54947fb75115cfe05bb54cc62c30+74   # input collection
 254           - 400f03b8c5d2dc3dcb513a21b626ef88+51   # md5sum output collection
 255           - 3738166916ca5f6f6ad12bf7e06b4a21+51   # rev output collection
 256           - bc37c17a37aa25229e5de1339b27fbcc+112  # runner output json
 257     out: [out, success]
 258     run: framework/testcase.cwl
 259
 260   twostep-remote-to-home:
 261     doc: |
 262       Two step workflow.  The runner is on the home cluster, the first
 263       step is on the remote cluster, the second step is on the home
 264       cluster.
 265     in:
 266       arvados_api_token: arvados_api_token
 267       arvado_api_host_insecure: arvado_api_host_insecure
 268       arvados_api_hosts: arvados_api_hosts
 269       arvados_cluster_ids: arvados_cluster_ids
 270       acr: acr
 271       wf:
 272         default:
 273           class: File
 274           location: cases/twostep-remote-to-home.cwl
 275           secondaryFiles:
 276             - class: File
 277               location: cases/md5sum.cwl
 278             - class: File
 279               location: cases/rev.cwl
 280       obj:
 281         default:
 282           inp:
 283             class: File
 284             location: data/twostep-remote-to-home.txt
 285         valueFrom: |-
 286           ${
 287           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 288           self["revCluster"] = inputs.arvados_cluster_ids[0];
 289           return self;
 290           }
 291       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 292       scrub_image: {default: "arvados/fed-test:twostep-remote-to-home"}
 293       scrub_collections:
 294         default:
 295           - cce89b9f7b6e163978144051ce5f071a+74   # input collection
 296           - 0c358c3af63644c6343766feff1b7238+51   # md5sum output collection
 297           - 33fb7d512bf21f04847eca58cea46e74+51   # rev output collection
 298           - 912e04aa3db04aba008cf5cd46c277b2+112  # runner output json
 299     out: [out, success]
 300     run: framework/testcase.cwl
 301
 302   twostep-both-remote:
 303     doc: |
 304       Two step workflow.  The runner is on the home cluster, both steps are
 305       on the remote cluster.
 306     in:
 307       arvados_api_token: arvados_api_token
 308       arvado_api_host_insecure: arvado_api_host_insecure
 309       arvados_api_hosts: arvados_api_hosts
 310       arvados_cluster_ids: arvados_cluster_ids
 311       acr: acr
 312       wf:
 313         default:
 314           class: File
 315           location: cases/twostep-both-remote.cwl
 316           secondaryFiles:
 317             - class: File
 318               location: cases/md5sum.cwl
 319             - class: File
 320               location: cases/rev.cwl
 321       obj:
 322         default:
 323           inp:
 324             class: File
 325             location: data/twostep-both-remote.txt
 326         valueFrom: |-
 327           ${
 328           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 329           self["revCluster"] = inputs.arvados_cluster_ids[1];
 330           return self;
 331           }
 332       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 333       scrub_image: {default: "arvados/fed-test:twostep-both-remote"}
 334       scrub_collections:
 335         default:
 336           - 3c5e39939cf197d304ac1eac20841238+71   # input collection
 337           - 3edb99aa607731593969cdab663d65b4+51   # md5sum output collection
 338           - a91625b7139e60fe61a88cae42fbee13+51   # rev output collection
 339           - ddfa58a81953dad08436d571615dd584+112  # runner output json
 340     out: [out, success]
 341     run: framework/testcase.cwl
 342
 343   twostep-remote-copy-to-home:
 344     doc: |
 345       Two step workflow.  The runner is on the home cluster, the first
 346       step is on the remote cluster, the second step is on the home
 347       cluster, and propagates its input file directly from input to
 348       output by symlinking the input file in the output directory.
 349       Tests that crunch-run will copy blocks from remote to local
 350       when preparing output collection.
 351     in:
 352       arvados_api_token: arvados_api_token
 353       arvado_api_host_insecure: arvado_api_host_insecure
 354       arvados_api_hosts: arvados_api_hosts
 355       arvados_cluster_ids: arvados_cluster_ids
 356       acr: acr
 357       wf:
 358         default:
 359           class: File
 360           location: cases/twostep-remote-copy-to-home.cwl
 361           secondaryFiles:
 362             - class: File
 363               location: cases/md5sum.cwl
 364             - class: File
 365               location: cases/rev-input-to-output.cwl
 366       obj:
 367         default:
 368           inp:
 369             class: File
 370             location: data/twostep-remote-copy-to-home.txt
 371         valueFrom: |-
 372           ${
 373           self["md5sumCluster"] = inputs.arvados_cluster_ids[1];
 374           self["revCluster"] = inputs.arvados_cluster_ids[0];
 375           return self;
 376           }
 377       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 378       scrub_image: {default: "arvados/fed-test:twostep-remote-copy-to-home"}
 379       scrub_collections:
 380         default:
 381           - 538887bc29a3098bf79abdb8536d17bd+79   # input collection
 382           - 14da0e0d52d7ab2945427074b275e9ee+51   # md5sum output collection
 383           - 2d3a4a840077390a0d7788f169eaba89+112  # rev output collection
 384           - 2d3a4a840077390a0d7788f169eaba89+112  # runner output json
 385     out: [out, success]
 386     run: framework/testcase.cwl
 387
 388   scatter-gather:
 389     doc: ""
 390     in:
 391       arvados_api_token: arvados_api_token
 392       arvado_api_host_insecure: arvado_api_host_insecure
 393       arvados_api_hosts: arvados_api_hosts
 394       arvados_cluster_ids: arvados_cluster_ids
 395       acr: acr
 396       wf:
 397         default:
 398           class: File
 399           location: cases/scatter-gather.cwl
 400           secondaryFiles:
 401             - class: File
 402               location: cases/md5sum.cwl
 403             - class: File
 404               location: cases/cat.cwl
 405       obj:
 406         default:
 407           shards:
 408             - class: File
 409               location: data/scatter-gather-s1.txt
 410             - class: File
 411               location: data/scatter-gather-s2.txt
 412             - class: File
 413               location: data/scatter-gather-s3.txt
 414         valueFrom: |-
 415           ${
 416           self["clusters"] = inputs.arvados_cluster_ids;
 417           return self;
 418           }
 419       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 420       scrub_image: {default: "arvados/fed-test:scatter-gather"}
 421       scrub_collections:
 422         default:
 423           - 99cc18329bce1b4a5fe6c4cf60477668+209  # input collection
 424           - 2e570e844e03c7027baad148642d726f+51   # s1 md5sum output collection
 425           - 61c88ee7811d0b849b5c06376eb065a6+51   # s2 md5sum output collection
 426           - 85aaf18d638045fe609e025d3a319b2a+51   # s3 md5sum output collection
 427           - ec44bcba77e65128f1a8f843d881ede4+56   # cat output collection
 428           - 89de265942800ae36549109969940363+117  # runner output json
 429     out: [out, success]
 430     run: framework/testcase.cwl
 431
 432   threestep-remote:
 433     doc: ""
 434     in:
 435       arvados_api_token: arvados_api_token
 436       arvado_api_host_insecure: arvado_api_host_insecure
 437       arvados_api_hosts: arvados_api_hosts
 438       arvados_cluster_ids: arvados_cluster_ids
 439       acr: acr
 440       wf:
 441         default:
 442           class: File
 443           location: cases/threestep-remote.cwl
 444           secondaryFiles:
 445             - class: File
 446               location: cases/md5sum.cwl
 447             - class: File
 448               location: cases/rev-input-to-output.cwl
 449       obj:
 450         default:
 451           inp:
 452             class: File
 453             location: data/threestep-remote.txt
 454         valueFrom: |-
 455           ${
 456           self["clusterA"] = inputs.arvados_cluster_ids[0];
 457           self["clusterB"] = inputs.arvados_cluster_ids[1];
 458           self["clusterC"] = inputs.arvados_cluster_ids[2];
 459           return self;
 460           }
 461       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 462       scrub_image: {default: "arvados/fed-test:threestep-remote"}
 463       scrub_collections:
 464         default:
 465           - 9fbf33e62876357fe134f619865cc5a5+68   # input collection
 466           - 210c5f2a716f6689b04316acd4928c10+51   # md5sum output collection
 467           - 3abea7506269d5ebf61fb17c78bbd2af+105  # revB output
 468           - 9e1b3acb28949759ad07e4c9740bbaa5+113  # revC output
 469           - 8c86dbec7de7948871b5e168ede417e1+120  # runner output json
 470     out: [out, success]
 471     run: framework/testcase.cwl
 472
 473   hint-on-wf:
 474     doc: |
 475       Single step workflow with the runner on the home cluster and the
 476       step on the remote cluster.  ClusterTarget hint is at the workflow level.
 477     in:
 478       arvados_api_token: arvados_api_token
 479       arvado_api_host_insecure: arvado_api_host_insecure
 480       arvados_api_hosts: arvados_api_hosts
 481       arvados_cluster_ids: arvados_cluster_ids
 482       acr: acr
 483       wf:
 484         default:
 485           class: File
 486           location: cases/hint-on-wf.cwl
 487           secondaryFiles:
 488             - class: File
 489               location: cases/md5sum.cwl
 490       obj:
 491         default:
 492           inp:
 493             class: File
 494             location: data/hint-on-wf.txt
 495         valueFrom: |-
 496           ${
 497           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 498           return self;
 499           }
 500       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 501       scrub_image: {default: "arvados/fed-test:hint-on-wf"}
 502       scrub_collections:
 503         default:
 504           - 862433f328041b2525c90b1dc3c462fd+62   # input collection
 505           - 9a68b0b9720977faba8a28e75a4398b7+51   # md5sum output collection
 506           - 6a601cddb36ee2f766783b1aa9ff8d66+112  # runner output json
 507     out: [out, success]
 508     run: framework/testcase.cwl
 509
 510   hint-on-tool:
 511     doc: |
 512       Single step workflow with the runner on the home cluster and the
 513       step on the remote cluster.  ClusterTarget hint is at the tool level.
 514     in:
 515       arvados_api_token: arvados_api_token
 516       arvado_api_host_insecure: arvado_api_host_insecure
 517       arvados_api_hosts: arvados_api_hosts
 518       arvados_cluster_ids: arvados_cluster_ids
 519       acr: acr
 520       wf:
 521         default:
 522           class: File
 523           location: cases/hint-on-tool.cwl
 524           secondaryFiles:
 525             - class: File
 526               location: cases/md5sum-tool-hint.cwl
 527       obj:
 528         default:
 529           inp:
 530             class: File
 531             location: data/hint-on-tool.txt
 532         valueFrom: |-
 533           ${
 534           self["runOnCluster"] = inputs.arvados_cluster_ids[1];
 535           return self;
 536           }
 537       runner_cluster: { valueFrom: "$(inputs.arvados_cluster_ids[0])" }
 538       scrub_image: {default: "arvados/fed-test:hint-on-tool"}
 539       scrub_collections:
 540         default:
 541           - 6803004a4f8db9f8d1d54f6229851599+64   # input collection
 542           - cacb0d56235564b5ff485c5b31215ab5+51   # md5sum output collection
 543           - 2b50af43fdd84a9e906be2d54b92cddf+112  # runner output json
 544     out: [out, success]
 545     run: framework/testcase.cwl