sdk/python/tests/fed-migrate/fed-migrate.cwl

   1 #!/usr/bin/env cwl-runner
   2 $graph:
   3   - class: Workflow
   4     cwlVersion: v1.0
   5     id: '#run_test'
   6     inputs:
   7       - id: arvados_api_hosts
   8         type:
   9           items: string
  10           type: array
  11       - id: superuser_tokens
  12         type:
  13           items: string
  14           type: array
  15       - default: arv-federation-migrate
  16         id: fed_migrate
  17         type: string
  18     outputs:
  19       - id: report3
  20         outputSource: main_2/report3
  21         type: File
  22     requirements:
  23       InlineJavascriptRequirement: {}
  24       MultipleInputFeatureRequirement: {}
  25       ScatterFeatureRequirement: {}
  26       StepInputExpressionRequirement: {}
  27       SubworkflowFeatureRequirement: {}
  28     steps:
  29       - id: main_1
  30         in:
  31           arvados_api_hosts:
  32             source: arvados_api_hosts
  33           create_users:
  34             default:
  35               class: File
  36               location: create_users.py
  37           superuser_tokens:
  38             source: superuser_tokens
  39         out:
  40           - ar
  41         run:
  42           arguments:
  43             - python
  44             - $(inputs.create_users)
  45             - _script
  46           class: CommandLineTool
  47           id: main_1_embed
  48           inputs:
  49             - id: arvados_api_hosts
  50               type:
  51                 items: string
  52                 type: array
  53             - id: superuser_tokens
  54               type:
  55                 items: string
  56                 type: array
  57             - id: create_users
  58               type: File
  59           outputs:
  60             - id: ar
  61               outputBinding:
  62                 outputEval: $(inputs.arvados_api_hosts)
  63               type:
  64                 items: string
  65                 type: array
  66           requirements:
  67             InitialWorkDirRequirement:
  68               listing:
  69                 - entry: |
  70                     {
  71                       "arvados_api_hosts": $(inputs.arvados_api_hosts),
  72                       "superuser_tokens": $(inputs.superuser_tokens)
  73                     }
  74                   entryname: _script
  75             InlineJavascriptRequirement: {}
  76       - id: main_2
  77         in:
  78           ar:
  79             source: main_1/ar
  80           arvados_api_hosts:
  81             source: arvados_api_hosts
  82           fed_migrate:
  83             source: fed_migrate
  84           host:
  85             valueFrom: '$(inputs.arvados_api_hosts[0])'
  86           superuser_tokens:
  87             source: superuser_tokens
  88           token:
  89             valueFrom: '$(inputs.superuser_tokens[0])'
  90         out:
  91           - report
  92           - report2
  93           - report3
  94           - r
  95         run:
  96           class: Workflow
  97           id: main_2_embed
  98           inputs:
  99             - id: ar
 100               type:
 101                 items: string
 102                 type: array
 103             - id: arvados_api_hosts
 104               type:
 105                 items: string
 106                 type: array
 107             - id: superuser_tokens
 108               type:
 109                 items: string
 110                 type: array
 111             - id: fed_migrate
 112               type: string
 113             - id: host
 114               type: Any
 115             - id: token
 116               type: Any
 117           outputs:
 118             - id: report
 119               outputSource: main_2_embed_1/report
 120               type: File
 121             - id: report2
 122               outputSource: main_2_embed_2/report2
 123               type: File
 124             - id: report3
 125               outputSource: main_2_embed_3/report3
 126               type: File
 127             - id: r
 128               outputSource: main_2_embed_4/r
 129               type: File
 130           requirements:
 131             - class: EnvVarRequirement
 132               envDef:
 133                 ARVADOS_API_HOST: $(inputs.host)
 134                 ARVADOS_API_HOST_INSECURE: '1'
 135                 ARVADOS_API_TOKEN: $(inputs.token)
 136           steps:
 137             - id: main_2_embed_1
 138               in:
 139                 fed_migrate:
 140                   source: fed_migrate
 141                 host:
 142                   source: host
 143                 token:
 144                   source: token
 145               out:
 146                 - report
 147               run:
 148                 arguments:
 149                   - $(inputs.fed_migrate)
 150                   - '--report'
 151                   - report.csv
 152                 class: CommandLineTool
 153                 id: main_2_embed_1_embed
 154                 inputs:
 155                   - id: fed_migrate
 156                     type: string
 157                   - id: host
 158                     type: Any
 159                   - id: token
 160                     type: Any
 161                 outputs:
 162                   - id: report
 163                     outputBinding:
 164                       glob: report.csv
 165                     type: File
 166                 requirements:
 167                   InlineJavascriptRequirement: {}
 168             - id: main_2_embed_2
 169               in:
 170                 host:
 171                   source: host
 172                 report:
 173                   source: main_2_embed_1/report
 174                 token:
 175                   source: token
 176               out:
 177                 - report2
 178               run:
 179                 arguments:
 180                   - sed
 181                   - '-E'
 182                   - 's/,(case[1-8])2?,/,\1,/g'
 183                 class: CommandLineTool
 184                 id: main_2_embed_2_embed
 185                 inputs:
 186                   - id: report
 187                     type: File
 188                   - id: host
 189                     type: Any
 190                   - id: token
 191                     type: Any
 192                 outputs:
 193                   - id: report2
 194                     outputBinding:
 195                       glob: report.csv
 196                     type: File
 197                 requirements:
 198                   InlineJavascriptRequirement: {}
 199                 stdin: $(inputs.report.path)
 200                 stdout: report.csv
 201             - id: main_2_embed_3
 202               in:
 203                 fed_migrate:
 204                   source: fed_migrate
 205                 host:
 206                   source: host
 207                 report2:
 208                   source: main_2_embed_2/report2
 209                 token:
 210                   source: token
 211               out:
 212                 - report3
 213               run:
 214                 arguments:
 215                   - $(inputs.fed_migrate)
 216                   - '--migrate'
 217                   - $(inputs.report2)
 218                 class: CommandLineTool
 219                 id: main_2_embed_3_embed
 220                 inputs:
 221                   - id: report2
 222                     type: File
 223                   - id: fed_migrate
 224                     type: string
 225                   - id: host
 226                     type: Any
 227                   - id: token
 228                     type: Any
 229                 outputs:
 230                   - id: report3
 231                     outputBinding:
 232                       outputEval: $(inputs.report2)
 233                     type: File
 234                 requirements:
 235                   InlineJavascriptRequirement: {}
 236             - id: main_2_embed_4
 237               in:
 238                 arvados_api_hosts:
 239                   source: arvados_api_hosts
 240                 check:
 241                   default:
 242                     class: File
 243                     location: check.py
 244                 host:
 245                   source: host
 246                 report3:
 247                   source: main_2_embed_3/report3
 248                 superuser_tokens:
 249                   source: superuser_tokens
 250                 token:
 251                   source: token
 252               out:
 253                 - r
 254               run:
 255                 arguments:
 256                   - python
 257                   - $(inputs.check)
 258                   - _script
 259                 class: CommandLineTool
 260                 id: main_2_embed_4_embed
 261                 inputs:
 262                   - id: report3
 263                     type: File
 264                   - id: host
 265                     type: Any
 266                   - id: token
 267                     type: Any
 268                   - id: arvados_api_hosts
 269                     type:
 270                       items: string
 271                       type: array
 272                   - id: superuser_tokens
 273                     type:
 274                       items: string
 275                       type: array
 276                   - id: check
 277                     type: File
 278                 outputs:
 279                   - id: r
 280                     outputBinding:
 281                       outputEval: $(inputs.report3)
 282                     type: File
 283                 requirements:
 284                   InitialWorkDirRequirement:
 285                     listing:
 286                       - entry: |
 287                           {
 288                             "arvados_api_hosts": $(inputs.arvados_api_hosts),
 289                             "superuser_tokens": $(inputs.superuser_tokens)
 290                           }
 291                         entryname: _script
 292                   InlineJavascriptRequirement: {}
 293   - arguments:
 294       - arvbox
 295       - cat
 296       - /var/lib/arvados/superuser_token
 297     class: CommandLineTool
 298     cwlVersion: v1.0
 299     id: '#superuser_tok'
 300     inputs:
 301       - id: container
 302         type: string
 303     outputs:
 304       - id: superuser_token
 305         outputBinding:
 306           glob: superuser_token.txt
 307           loadContents: true
 308           outputEval: '$(self[0].contents.trim())'
 309         type: string
 310     requirements:
 311       EnvVarRequirement:
 312         envDef:
 313           ARVBOX_CONTAINER: $(inputs.container)
 314       InlineJavascriptRequirement: {}
 315     stdout: superuser_token.txt
 316   - class: Workflow
 317     id: '#main'
 318     inputs:
 319       - id: arvados_api_hosts
 320         type:
 321           items: string
 322           type: array
 323       - id: arvados_cluster_ids
 324         type:
 325           items: string
 326           type: array
 327       - id: superuser_tokens
 328         type:
 329           items: string
 330           type: array
 331       - id: arvbox_containers
 332         type:
 333           items: string
 334           type: array
 335       - default: arv-federation-migrate
 336         id: fed_migrate
 337         type: string
 338       - id: arvbox_bin
 339         type: File
 340       - default: 15531-logincluster-migrate
 341         id: refspec
 342         type: string
 343     outputs:
 344       - id: supertok
 345         outputSource: main_2/supertok
 346         type:
 347           items: string
 348           type: array
 349       - id: report
 350         outputSource: run_test_3/report3
 351         type: File
 352     requirements:
 353       InlineJavascriptRequirement: {}
 354       MultipleInputFeatureRequirement: {}
 355       ScatterFeatureRequirement: {}
 356       StepInputExpressionRequirement: {}
 357       SubworkflowFeatureRequirement: {}
 358     steps:
 359       - id: main_1
 360         in:
 361           arvados_cluster_ids:
 362             source: arvados_cluster_ids
 363         out:
 364           - logincluster
 365         run:
 366           class: ExpressionTool
 367           expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
 368           inputs:
 369             - id: arvados_cluster_ids
 370               type:
 371                 items: string
 372                 type: array
 373           outputs:
 374             - id: logincluster
 375               type: string
 376       - id: main_2
 377         in:
 378           arvbox_bin:
 379             source: arvbox_bin
 380           cluster_id:
 381             source: arvados_cluster_ids
 382           container:
 383             source: arvbox_containers
 384           host:
 385             source: arvados_api_hosts
 386           logincluster:
 387             source: main_1/logincluster
 388           refspec:
 389             source: refspec
 390         out:
 391           - supertok
 392         run:
 393           class: Workflow
 394           id: main_2_embed
 395           inputs:
 396             - id: container
 397               type: string
 398             - id: cluster_id
 399               type: string
 400             - id: host
 401               type: string
 402             - id: logincluster
 403               type: string
 404             - id: arvbox_bin
 405               type: File
 406             - id: refspec
 407               type: string
 408           outputs:
 409             - id: supertok
 410               outputSource: superuser_tok_3/superuser_token
 411               type: string
 412           requirements:
 413             - class: EnvVarRequirement
 414               envDef:
 415                 ARVBOX_CONTAINER: $(inputs.container)
 416           steps:
 417             - id: main_2_embed_1
 418               in:
 419                 cluster_id:
 420                   source: cluster_id
 421                 container:
 422                   source: container
 423                 logincluster:
 424                   source: logincluster
 425                 set_login:
 426                   default:
 427                     class: File
 428                     location: set_login.py
 429               out:
 430                 - c
 431               run:
 432                 arguments:
 433                   - sh
 434                   - _script
 435                 class: CommandLineTool
 436                 id: main_2_embed_1_embed
 437                 inputs:
 438                   - id: container
 439                     type: string
 440                   - id: cluster_id
 441                     type: string
 442                   - id: logincluster
 443                     type: string
 444                   - id: set_login
 445                     type: File
 446                 outputs:
 447                   - id: c
 448                     outputBinding:
 449                       outputEval: $(inputs.container)
 450                     type: string
 451                 requirements:
 452                   InitialWorkDirRequirement:
 453                     listing:
 454                       - entry: >
 455                           set -x
 456
 457                           docker cp
 458                           $(inputs.container):/var/lib/arvados/cluster_config.yml.override
 459                           .
 460
 461                           chmod +w cluster_config.yml.override
 462
 463                           python $(inputs.set_login.path)
 464                           cluster_config.yml.override $(inputs.cluster_id)
 465                           $(inputs.logincluster)
 466
 467                           docker cp cluster_config.yml.override
 468                           $(inputs.container):/var/lib/arvados
 469                         entryname: _script
 470                   InlineJavascriptRequirement: {}
 471             - id: main_2_embed_2
 472               in:
 473                 arvbox_bin:
 474                   source: arvbox_bin
 475                 c:
 476                   source: main_2_embed_1/c
 477                 container:
 478                   source: container
 479                 host:
 480                   source: host
 481                 refspec:
 482                   source: refspec
 483               out:
 484                 - d
 485               run:
 486                 arguments:
 487                   - sh
 488                   - _script
 489                 class: CommandLineTool
 490                 id: main_2_embed_2_embed
 491                 inputs:
 492                   - id: container
 493                     type: string
 494                   - id: host
 495                     type: string
 496                   - id: arvbox_bin
 497                     type: File
 498                   - id: c
 499                     type: string
 500                   - id: refspec
 501                     type: string
 502                 outputs:
 503                   - id: d
 504                     outputBinding:
 505                       outputEval: $(inputs.c)
 506                     type: string
 507                 requirements:
 508                   InitialWorkDirRequirement:
 509                     listing:
 510                       - entry: >
 511                           set -xe
 512
 513                           $(inputs.arvbox_bin.path) pipe <<EOF
 514
 515                           cd /usr/src/arvados
 516
 517                           git fetch
 518
 519                           git checkout -f $(inputs.refspec)
 520
 521                           git pull
 522
 523                           chown arvbox:arvbox -R .
 524
 525                           EOF
 526
 527
 528                           $(inputs.arvbox_bin.path) hotreset
 529
 530
 531                           while ! curl --fail --insecure --silent
 532                           https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
 533                           >/dev/null ; do sleep 3 ; done
 534
 535                           export ARVADOS_API_HOST=$(inputs.host)
 536
 537                           export ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path)
 538                           cat /var/lib/arvados/superuser_token)
 539
 540                           export ARVADOS_API_HOST_INSECURE=1
 541
 542                           ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
 543                           cat /var/lib/arvados/vm-uuid)
 544
 545                           while ! python -c "import arvados ;
 546                           arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
 547                           2>/dev/null ; do sleep 3; done
 548                         entryname: _script
 549                   InlineJavascriptRequirement: {}
 550             - id: superuser_tok_3
 551               in:
 552                 container:
 553                   source: container
 554                 d:
 555                   source: main_2_embed_2/d
 556               out:
 557                 - superuser_token
 558               run: '#superuser_tok'
 559         scatter:
 560           - container
 561           - cluster_id
 562           - host
 563         scatterMethod: dotproduct
 564       - id: run_test_3
 565         in:
 566           arvados_api_hosts:
 567             source: arvados_api_hosts
 568           fed_migrate:
 569             source: fed_migrate
 570           superuser_tokens:
 571             source: main_2/supertok
 572         out:
 573           - report3
 574         run: '#run_test'
 575 cwlVersion: v1.0
 576