X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/021d36f17fe0329e869324d3764eaaf15c3a0771..0e3fc1c56dc4b0aea228dc3de045e218749de2e3:/build/run-build-packages.sh

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7763ca5454..915a331935 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
 
 . `dirname "$(readlink -f "$0")"`/run-library.sh
 . `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
@@ -104,6 +107,9 @@ case "$TARGET" in
     debian8)
         FORMAT=deb
         ;;
+    debian9)
+        FORMAT=deb
+        ;;
     ubuntu1204)
         FORMAT=deb
         ;;
@@ -214,7 +220,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
         make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
         fpm_build install/lib/=/usr/share libarvados-perl \
         "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
-        "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+        "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
         mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
   fi
 fi
@@ -334,6 +340,7 @@ fi
 # Go binaries
 cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
+go get github.com/kardianos/govendor
 package_go_binary sdk/go/crunchrunner crunchrunner \
     "Crunchrunner executes a command inside a container and uploads the output"
 package_go_binary services/arv-git-httpd arvados-git-httpd \
@@ -346,6 +353,8 @@ package_go_binary services/crunch-run crunch-run \
     "Supervise a single Crunch container"
 package_go_binary services/crunchstat crunchstat \
     "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary services/health arvados-health \
+    "Check health of all Arvados cluster services"
 package_go_binary services/keep-balance keep-balance \
     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
 package_go_binary services/keepproxy keepproxy \
@@ -356,6 +365,8 @@ package_go_binary services/keep-web keep-web \
     "Static web hosting service for user data stored in Arvados Keep"
 package_go_binary services/ws arvados-ws \
     "Arvados Websocket server"
+package_go_binary tools/sync-groups arvados-sync-groups \
+    "Synchronize remote groups into Arvados from an external source"
 package_go_binary tools/keep-block-check keep-block-check \
     "Verify that all data from one set of Keep servers to another was copied"
 package_go_binary tools/keep-rsync keep-rsync \
@@ -382,10 +393,16 @@ fi
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/sdk/cwl/build"
 arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
-arvados_cwl_runner_iteration=3
+declare -a iterargs=()
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+    arvados_cwl_runner_iteration=3
+    iterargs+=(--iteration $arvados_cwl_runner_iteration)
+else
+    arvados_cwl_runner_iteration=
+fi
 test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
 if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+  fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
 fi
 
 # schema_salad. This is a python dependency of arvados-cwl-runner,
@@ -404,16 +421,16 @@ fi
 #
 # Ward, 2016-03-17
 saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
 if [[ "$?" == "0" ]]; then
-  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2"
+  fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
 fi
 
 # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
 cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
 if [[ "$?" == "0" ]]; then
-  fpm_build cwltool "" "" python $cwltoolversion
+  fpm_build cwltool "" "" python $cwltoolversion --iteration 2
 fi
 
 # The PAM module
@@ -451,23 +468,25 @@ fi
 cd $WORKSPACE/packages/$TARGET
 rm -rf "$WORKSPACE/services/dockercleaner/build"
 dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-dockercleaner_iteration=3
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
 if [[ "$?" == "0" ]]; then
-  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+  fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
 fi
 
 # The Arvados crunchstat-summary tool
 cd $WORKSPACE/packages/$TARGET
 crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
 if [[ "$?" == "0" ]]; then
   rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
-  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+  fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
 fi
 
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
-  # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
   LIBCLOUD_DIR=$(mktemp -d)
   (
       cd $LIBCLOUD_DIR
@@ -479,7 +498,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONL
       handle_python_package
       DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
   )
-  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+  fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
   rm -rf $LIBCLOUD_DIR
 fi
 
@@ -601,7 +620,7 @@ if [[ "$?" == "0" ]] ; then
 
       # We need to bundle to be ready even when we build a package without vendor directory
       # because asset compilation requires it.
-      bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+      bundle install --system >"$STDOUT_IF_DEBUG"
 
       # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
       # and we want that in the package, so it's easier to not exclude the tmp directory
@@ -613,7 +632,9 @@ if [[ "$?" == "0" ]] ; then
       \cp config/application.yml.example config/application.yml -f
       \cp config/environments/production.rb.example config/environments/production.rb -f
       sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
+      sed -i 's/keep_web_url: false/keep_web_url: exampledotcom/' config/application.yml
 
+      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >/dev/null
       RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
 
       # Remove generated configuration files so they don't go in the package.