sdk/python/tests/fed-migrate/fed-migrate.cwl

   1 #!/usr/bin/env cwl-runner
   2 $graph:
   3   - class: Workflow
   4     cwlVersion: v1.0
   5     id: '#run_test'
   6     inputs:
   7       - id: arvados_api_hosts
   8         type:
   9           items: string
  10           type: array
  11       - id: superuser_tokens
  12         type:
  13           items: string
  14           type: array
  15       - default: arv-federation-migrate
  16         id: fed_migrate
  17         type: string
  18     outputs:
  19       - id: report3
  20         outputSource: main_2/report3
  21         type: File
  22     requirements:
  23       InlineJavascriptRequirement: {}
  24       MultipleInputFeatureRequirement: {}
  25       ScatterFeatureRequirement: {}
  26       StepInputExpressionRequirement: {}
  27       SubworkflowFeatureRequirement: {}
  28     steps:
  29       - id: main_1
  30         in:
  31           arvados_api_hosts:
  32             source: arvados_api_hosts
  33           create_users:
  34             default:
  35               class: File
  36               location: create_users.py
  37           superuser_tokens:
  38             source: superuser_tokens
  39         out:
  40           - ar
  41         run:
  42           arguments:
  43             - python
  44             - $(inputs.create_users)
  45             - _script
  46           class: CommandLineTool
  47           id: main_1_embed
  48           inputs:
  49             - id: arvados_api_hosts
  50               type:
  51                 items: string
  52                 type: array
  53             - id: superuser_tokens
  54               type:
  55                 items: string
  56                 type: array
  57             - id: create_users
  58               type: File
  59           outputs:
  60             - id: ar
  61               outputBinding:
  62                 outputEval: $(inputs.arvados_api_hosts)
  63               type:
  64                 items: string
  65                 type: array
  66           requirements:
  67             InitialWorkDirRequirement:
  68               listing:
  69                 - entry: |
  70                     {
  71                       "arvados_api_hosts": $(inputs.arvados_api_hosts),
  72                       "superuser_tokens": $(inputs.superuser_tokens)
  73                     }
  74                   entryname: _script
  75             InlineJavascriptRequirement: {}
  76       - id: main_2
  77         in:
  78           ar:
  79             source: main_1/ar
  80           arvados_api_hosts:
  81             source: arvados_api_hosts
  82           fed_migrate:
  83             source: fed_migrate
  84           host:
  85             valueFrom: '$(inputs.arvados_api_hosts[0])'
  86           superuser_tokens:
  87             source: superuser_tokens
  88           token:
  89             valueFrom: '$(inputs.superuser_tokens[0])'
  90         out:
  91           - report
  92           - report2
  93           - report3
  94           - r
  95         run:
  96           class: Workflow
  97           id: main_2_embed
  98           inputs:
  99             - id: ar
 100               type:
 101                 items: string
 102                 type: array
 103             - id: arvados_api_hosts
 104               type:
 105                 items: string
 106                 type: array
 107             - id: superuser_tokens
 108               type:
 109                 items: string
 110                 type: array
 111             - id: fed_migrate
 112               type: string
 113             - id: host
 114               type: Any
 115             - id: token
 116               type: Any
 117           outputs:
 118             - id: report
 119               outputSource: main_2_embed_1/report
 120               type: File
 121             - id: report2
 122               outputSource: main_2_embed_2/report2
 123               type: File
 124             - id: report3
 125               outputSource: main_2_embed_3/report3
 126               type: File
 127             - id: r
 128               outputSource: main_2_embed_4/r
 129               type: File
 130           requirements:
 131             - class: EnvVarRequirement
 132               envDef:
 133                 ARVADOS_API_HOST: $(inputs.host)
 134                 ARVADOS_API_TOKEN: $(inputs.token)
 135           steps:
 136             - id: main_2_embed_1
 137               in:
 138                 fed_migrate:
 139                   source: fed_migrate
 140                 host:
 141                   source: host
 142                 token:
 143                   source: token
 144               out:
 145                 - report
 146               run:
 147                 arguments:
 148                   - $(inputs.fed_migrate)
 149                   - '--report'
 150                   - report.csv
 151                 class: CommandLineTool
 152                 id: main_2_embed_1_embed
 153                 inputs:
 154                   - id: fed_migrate
 155                     type: string
 156                   - id: host
 157                     type: Any
 158                   - id: token
 159                     type: Any
 160                 outputs:
 161                   - id: report
 162                     outputBinding:
 163                       glob: report.csv
 164                     type: File
 165                 requirements:
 166                   InlineJavascriptRequirement: {}
 167             - id: main_2_embed_2
 168               in:
 169                 host:
 170                   source: host
 171                 report:
 172                   source: main_2_embed_1/report
 173                 token:
 174                   source: token
 175               out:
 176                 - report2
 177               run:
 178                 arguments:
 179                   - sed
 180                   - '-E'
 181                   - 's/,(case[1-8])2?,/,\1,/g'
 182                 class: CommandLineTool
 183                 id: main_2_embed_2_embed
 184                 inputs:
 185                   - id: report
 186                     type: File
 187                   - id: host
 188                     type: Any
 189                   - id: token
 190                     type: Any
 191                 outputs:
 192                   - id: report2
 193                     outputBinding:
 194                       glob: report.csv
 195                     type: File
 196                 requirements:
 197                   InlineJavascriptRequirement: {}
 198                 stdin: $(inputs.report.path)
 199                 stdout: report.csv
 200             - id: main_2_embed_3
 201               in:
 202                 fed_migrate:
 203                   source: fed_migrate
 204                 host:
 205                   source: host
 206                 report2:
 207                   source: main_2_embed_2/report2
 208                 token:
 209                   source: token
 210               out:
 211                 - report3
 212               run:
 213                 arguments:
 214                   - $(inputs.fed_migrate)
 215                   - '--migrate'
 216                   - $(inputs.report2)
 217                 class: CommandLineTool
 218                 id: main_2_embed_3_embed
 219                 inputs:
 220                   - id: report2
 221                     type: File
 222                   - id: fed_migrate
 223                     type: string
 224                   - id: host
 225                     type: Any
 226                   - id: token
 227                     type: Any
 228                 outputs:
 229                   - id: report3
 230                     outputBinding:
 231                       outputEval: $(inputs.report2)
 232                     type: File
 233                 requirements:
 234                   InlineJavascriptRequirement: {}
 235             - id: main_2_embed_4
 236               in:
 237                 arvados_api_hosts:
 238                   source: arvados_api_hosts
 239                 check:
 240                   default:
 241                     class: File
 242                     location: check.py
 243                 host:
 244                   source: host
 245                 report3:
 246                   source: main_2_embed_3/report3
 247                 superuser_tokens:
 248                   source: superuser_tokens
 249                 token:
 250                   source: token
 251               out:
 252                 - r
 253               run:
 254                 arguments:
 255                   - python
 256                   - $(inputs.check)
 257                   - _script
 258                 class: CommandLineTool
 259                 id: main_2_embed_4_embed
 260                 inputs:
 261                   - id: report3
 262                     type: File
 263                   - id: host
 264                     type: Any
 265                   - id: token
 266                     type: Any
 267                   - id: arvados_api_hosts
 268                     type:
 269                       items: string
 270                       type: array
 271                   - id: superuser_tokens
 272                     type:
 273                       items: string
 274                       type: array
 275                   - id: check
 276                     type: File
 277                 outputs:
 278                   - id: r
 279                     outputBinding:
 280                       outputEval: $(inputs.report3)
 281                     type: File
 282                 requirements:
 283                   InitialWorkDirRequirement:
 284                     listing:
 285                       - entry: |
 286                           {
 287                             "arvados_api_hosts": $(inputs.arvados_api_hosts),
 288                             "superuser_tokens": $(inputs.superuser_tokens)
 289                           }
 290                         entryname: _script
 291                   InlineJavascriptRequirement: {}
 292   - arguments:
 293       - arvbox
 294       - cat
 295       - /var/lib/arvados/superuser_token
 296     class: CommandLineTool
 297     cwlVersion: v1.0
 298     id: '#superuser_tok'
 299     inputs:
 300       - id: container
 301         type: string
 302     outputs:
 303       - id: superuser_token
 304         outputBinding:
 305           glob: superuser_token.txt
 306           loadContents: true
 307           outputEval: '$(self[0].contents.trim())'
 308         type: string
 309     requirements:
 310       EnvVarRequirement:
 311         envDef:
 312           ARVBOX_CONTAINER: $(inputs.container)
 313       InlineJavascriptRequirement: {}
 314     stdout: superuser_token.txt
 315   - class: Workflow
 316     id: '#main'
 317     inputs:
 318       - id: arvados_api_hosts
 319         type:
 320           items: string
 321           type: array
 322       - id: arvados_cluster_ids
 323         type:
 324           items: string
 325           type: array
 326       - id: superuser_tokens
 327         type:
 328           items: string
 329           type: array
 330       - id: arvbox_containers
 331         type:
 332           items: string
 333           type: array
 334       - default: arv-federation-migrate
 335         id: fed_migrate
 336         type: string
 337     outputs:
 338       - id: supertok
 339         outputSource: main_2/supertok
 340         type:
 341           items: string
 342           type: array
 343       - id: report
 344         outputSource: run_test_3/report3
 345         type: File
 346     requirements:
 347       InlineJavascriptRequirement: {}
 348       MultipleInputFeatureRequirement: {}
 349       ScatterFeatureRequirement: {}
 350       StepInputExpressionRequirement: {}
 351       SubworkflowFeatureRequirement: {}
 352     steps:
 353       - id: main_1
 354         in:
 355           arvados_cluster_ids:
 356             source: arvados_cluster_ids
 357         out:
 358           - logincluster
 359         run:
 360           class: ExpressionTool
 361           expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
 362           inputs:
 363             - id: arvados_cluster_ids
 364               type:
 365                 items: string
 366                 type: array
 367           outputs:
 368             - id: logincluster
 369               type: string
 370       - id: main_2
 371         in:
 372           cluster_id:
 373             source: arvados_cluster_ids
 374           container:
 375             source: arvbox_containers
 376           host:
 377             source: arvados_api_hosts
 378           logincluster:
 379             source: main_1/logincluster
 380         out:
 381           - supertok
 382         run:
 383           class: Workflow
 384           id: main_2_embed
 385           inputs:
 386             - id: container
 387               type: string
 388             - id: cluster_id
 389               type: string
 390             - id: host
 391               type: string
 392             - id: logincluster
 393               type: string
 394           outputs:
 395             - id: supertok
 396               outputSource: superuser_tok_3/superuser_token
 397               type: string
 398           requirements:
 399             - class: EnvVarRequirement
 400               envDef:
 401                 ARVBOX_CONTAINER: $(inputs.container)
 402           steps:
 403             - id: main_2_embed_1
 404               in:
 405                 cluster_id:
 406                   source: cluster_id
 407                 container:
 408                   source: container
 409                 logincluster:
 410                   source: logincluster
 411                 set_login:
 412                   default:
 413                     class: File
 414                     location: set_login.py
 415               out:
 416                 - c
 417               run:
 418                 arguments:
 419                   - sh
 420                   - _script
 421                 class: CommandLineTool
 422                 id: main_2_embed_1_embed
 423                 inputs:
 424                   - id: container
 425                     type: string
 426                   - id: cluster_id
 427                     type: string
 428                   - id: logincluster
 429                     type: string
 430                   - id: set_login
 431                     type: File
 432                 outputs:
 433                   - id: c
 434                     outputBinding:
 435                       outputEval: $(inputs.container)
 436                     type: string
 437                 requirements:
 438                   InitialWorkDirRequirement:
 439                     listing:
 440                       - entry: >
 441                           set -x
 442
 443                           docker cp
 444                           $(inputs.container):/var/lib/arvados/cluster_config.yml.override
 445                           .
 446
 447                           chmod +w cluster_config.yml.override
 448
 449                           python $(inputs.set_login.path)
 450                           cluster_config.yml.override $(inputs.cluster_id)
 451                           $(inputs.logincluster)
 452
 453                           docker cp cluster_config.yml.override
 454                           $(inputs.container):/var/lib/arvados
 455                         entryname: _script
 456                   InlineJavascriptRequirement: {}
 457             - id: main_2_embed_2
 458               in:
 459                 c:
 460                   source: main_2_embed_1/c
 461                 container:
 462                   source: container
 463                 host:
 464                   source: host
 465               out:
 466                 - d
 467               run:
 468                 arguments:
 469                   - sh
 470                   - _script
 471                 class: CommandLineTool
 472                 id: main_2_embed_2_embed
 473                 inputs:
 474                   - id: container
 475                     type: string
 476                   - id: host
 477                     type: string
 478                   - id: c
 479                     type: string
 480                 outputs:
 481                   - id: d
 482                     outputBinding:
 483                       outputEval: $(inputs.c)
 484                     type: string
 485                 requirements:
 486                   InitialWorkDirRequirement:
 487                     listing:
 488                       - entry: >
 489                           set -x
 490
 491                           arvbox hotreset
 492
 493                           while ! curl --fail --insecure --silent
 494                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
 495                           >/dev/null ; do sleep 3 ; done
 496
 497                           export ARVADOS_API_HOST=$(inputs.host)
 498
 499                           export ARVADOS_API_TOKEN=\$(arvbox cat
 500                           /var/lib/arvados/superuser_token)
 501
 502                           export ARVADOS_API_HOST_INSECURE=1
 503
 504                           ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
 505                           /var/lib/arvados/vm-uuid)
 506
 507                           while ! python -c "import arvados ;
 508                           arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
 509                           2>/dev/null ; do sleep 3; done
 510                         entryname: _script
 511                   InlineJavascriptRequirement: {}
 512             - id: superuser_tok_3
 513               in:
 514                 container:
 515                   source: container
 516                 d:
 517                   source: main_2_embed_2/d
 518               out:
 519                 - superuser_token
 520               run: '#superuser_tok'
 521         scatter:
 522           - container
 523           - cluster_id
 524           - host
 525         scatterMethod: dotproduct
 526       - id: run_test_3
 527         in:
 528           arvados_api_hosts:
 529             source: arvados_api_hosts
 530           fed_migrate:
 531             source: fed_migrate
 532           superuser_tokens:
 533             source: main_2/supertok
 534         out:
 535           - report3
 536         run: '#run_test'
 537 cwlVersion: v1.0
 538