14360: Merge branch 'master' into 14360-dispatch-cloud

refs #14360

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --combined .licenseignore
index a0127cfa30f09a986b4dad176ae9358e573209fc,83c81b2fc21ca1db86206d82c2f50cae297b9b9c..06519a98e8bc45afcebdad584198a6b6bb47bf71
--- 1/.licenseignore
--- 2/.licenseignore
+++ b/.licenseignore
@@@ -13,6 -13,7 +13,7 @@@ build/package-test-dockerfiles/ubuntu16
  *by-sa-3.0.txt
  *COPYING
  doc/fonts/*
+ doc/user/cwl/federated/*
  */docker_image
  docker/jobs/apt.arvados.org.list
  */en.bootstrap.yml
@@@ -46,6 -47,7 +47,7 @@@
  */script/rails
  sdk/cwl/tests/input/blorp.txt
  sdk/cwl/tests/tool/blub.txt
+ sdk/cwl/tests/federation/data/*
  sdk/go/manifest/testdata/*_manifest
  sdk/java/.classpath
  sdk/java/pom.xml
@@@ -69,4 -71,4 +71,5 @@@ sdk/R/NAMESPAC
  sdk/R/.Rbuildignore
  sdk/R/ArvadosR.Rproj
  *.Rd
 +lib/dispatchcloud/test/sshkey_*
+ *.asc

diff --combined build/run-build-packages.sh
index 1b486b40b6ba8e3beb7899b39f0bb1892234bb31,73e2e628631bbc71ea84388b37e6ae4d60917734..f316c563bd53e1ea6ddac44ca0928c6b299d8ffe
--- 1/build/run-build-packages.sh
--- 2/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@@ -294,9 -294,6 +294,9 @@@ package_go_binary cmd/arvados-server ar
      "Arvados server daemons"
  package_go_binary cmd/arvados-server arvados-controller \
      "Arvados cluster controller daemon"
 +# No package until #14325
 +#package_go_binary cmd/arvados-server crunch-dispatch-cloud \
 +#    "Arvados cluster cloud dispatch"
  package_go_binary sdk/go/crunchrunner crunchrunner \
      "Crunchrunner executes a command inside a container and uploads the output"
  package_go_binary services/arv-git-httpd arvados-git-httpd \
@@@ -330,6 -327,28 +330,28 @@@ package_go_binary tools/keep-rsync keep
  package_go_binary tools/keep-exercise keep-exercise \
      "Performance testing tool for Arvados Keep"
  
+ 
+ # we need explicit debian_revision values in the dependencies for ruamel.yaml, because we have a package iteration
+ # greater than zero. So we parse setup.py, get the ruamel.yaml dependencies, tell fpm not to automatically include
+ # them in the package being built, and re-add them manually with an appropriate debian_revision value.
+ # See #14552 for the reason for this (nasty) workaround. We use ${ruamel_depends[@]} in a few places further down
+ # in this script.
+ # Ward, 2018-11-28
+ IFS=', ' read -r -a deps <<< `grep ruamel.yaml $WORKSPACE/sdk/python/setup.py |cut -f 3 -dl |sed -e "s/'//g"`
+ declare -a ruamel_depends=()
+ for i in ${deps[@]}; do
+   i=`echo "$i" | sed -e 's!\([0-9]\)! \1!'`
+   if [[ $i =~ .*\>.* ]]; then
+     ruamel_depends+=(--depends "python-ruamel.yaml $i-1")
+   elif [[ $i =~ .*\<.* ]]; then
+     ruamel_depends+=(--depends "python-ruamel.yaml $i-9")
+   else
+     echo "Encountered ruamel dependency that I can't parse. Aborting..."
+     exit 1
+   fi
+ done
+ 
+ 
  # The Python SDK
  # Please resist the temptation to add --no-python-fix-name to the fpm call here
  # (which would remove the python- prefix from the package name), because this
@@@ -342,7 -361,8 +364,8 @@@ rm -rf "$WORKSPACE/sdk/python/build
  arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)}
  test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python
  if [[ "$?" == "0" ]]; then
-   fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
+ 
+   fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git  --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
  fi
  
  # cwl-runner
@@@ -358,7 -378,7 +381,7 @@@ els
  fi
  test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
  if [[ "$?" == "0" ]]; then
-   fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" "${iterargs[@]}"
+   fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}" "${iterargs[@]}"
  fi
  
  # schema_salad. This is a python dependency of arvados-cwl-runner,
@@@ -384,9 -404,9 +407,9 @@@ f
  
  # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
  cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
- test_package_presence python-cwltool "$cwltoolversion" python 2
+ test_package_presence python-cwltool "$cwltoolversion" python 3
  if [[ "$?" == "0" ]]; then
-   fpm_build cwltool "" "" python $cwltoolversion --iteration 2
+   fpm_build cwltool "" "" python $cwltoolversion --iteration 3 --python-disable-dependency ruamel.yaml "${ruamel_depends[@]}"
  fi
  
  # The PAM module