X-Git-Url: https://git.arvados.org/arvados-dev.git/blobdiff_plain/bc27dd23b329c7d82deebb3065917dceba18da27..4735c45083770bf8344dd80dd84695bb4e0fd169:/jenkins/run-build-packages.sh diff --git a/jenkins/run-build-packages.sh b/jenkins/run-build-packages.sh index ff9acf6..c6e658c 100755 --- a/jenkins/run-build-packages.sh +++ b/jenkins/run-build-packages.sh @@ -95,9 +95,9 @@ case "$TARGET" in PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py \ - pykka requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname llfuse \ + 'pycurl<7.21.5') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; debian8) @@ -108,9 +108,9 @@ case "$TARGET" in PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py \ - pykka requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname llfuse \ + 'pycurl<7.21.5') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; ubuntu1204) @@ -121,9 +121,9 @@ case "$TARGET" in PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py \ - pykka requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse) + rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname llfuse \ + 'pycurl<7.21.5') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) ;; ubuntu1404) @@ -134,7 +134,7 @@ case "$TARGET" in PYTHON3_PKG_PREFIX=python3 PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse ciso8601 \ google-api-python-client six uritemplate oauth2client httplib2 \ - rsa pycurl backports.ssl_match_hostname) + rsa 'pycurl<7.21.5' backports.ssl_match_hostname) PYTHON3_BACKPORTS=(docker-py requests websocket-client) ;; centos6) @@ -145,10 +145,9 @@ case "$TARGET" in PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \ oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py \ - pykka requests six pyexecjs jsonschema \ - ciso8601 pycrypto backports.ssl_match_hostname pycurl - python-daemon lockfile llfuse) + rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \ + ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \ + python-daemon lockfile llfuse 'pbr<1.0') PYTHON3_BACKPORTS=(docker-py six requests websocket-client) export PYCURL_SSL_LIBRARY=nss ;; @@ -231,8 +230,9 @@ rm -rf install perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ fpm_build install/lib/=/usr/share libarvados-perl \ - "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \ - mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" + "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \ + "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \ + mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" # Ruby gems debug_echo -e "\nRuby gems\n" @@ -257,6 +257,9 @@ handle_python_package cd "$WORKSPACE/sdk/python" handle_python_package +cd "$WORKSPACE/sdk/cwl" +handle_python_package + cd "$WORKSPACE/services/fuse" handle_python_package @@ -285,77 +288,25 @@ handle_python_package cd $WORKSPACE/packages/$TARGET fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" - rm -r "$SRC_BUILD_DIR" + rm -rf "$SRC_BUILD_DIR" ) -# Keep +# Go binaries export GOPATH=$(mktemp -d) -mkdir -p "$GOPATH/src/git.curoverse.com" -ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" - -# keepstore -cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore" -PKG_VERSION=$(version_from_git) -go get "git.curoverse.com/arvados.git/services/keepstore" -cd $WORKSPACE/packages/$TARGET -fpm_build $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN" - -# Get GO SDK version -cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go" -GO_SDK_VERSION=$(version_from_git) -GO_SDK_TIMESTAMP=$(timestamp_from_git) - -# keepproxy -cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepproxy" -KEEPPROXY_VERSION=$(version_from_git) -KEEPPROXY_TIMESTAMP=$(timestamp_from_git) - -if [[ "$GO_SDK_TIMESTAMP" -gt "$KEEPPROXY_TIMESTAMP" ]]; then - PKG_VERSION=$GO_SDK_VERSION -else - PKG_VERSION=$KEEPPROXY_VERSION -fi - -go get "git.curoverse.com/arvados.git/services/keepproxy" -cd $WORKSPACE/packages/$TARGET -fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN" - -# datamanager -cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager" -DATAMANAGER_VERSION=$(version_from_git) -DATAMANAGER_TIMESTAMP=$(timestamp_from_git) - -if [[ "$GO_SDK_TIMESTAMP" -gt "$DATAMANAGER_TIMESTAMP" ]]; then - PKG_VERSION=$GO_SDK_VERSION -else - PKG_VERSION=$DATAMANAGER_VERSION -fi - -go get "git.curoverse.com/arvados.git/services/datamanager" -cd $WORKSPACE/packages/$TARGET -fpm_build $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed." - -# arv-git-httpd -cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd" -ARVGITHTTPD_VERSION=$(version_from_git) -ARVGITHTTPD_TIMESTAMP=$(timestamp_from_git) - -if [[ "$GO_SDK_TIMESTAMP" -gt "$ARVGITHTTPD_TIMESTAMP" ]]; then - PKG_VERSION=$GO_SDK_VERSION -else - PKG_VERSION=$ARVGITHTTPD_VERSION -fi - -go get "git.curoverse.com/arvados.git/services/arv-git-httpd" -cd $WORKSPACE/packages/$TARGET -fpm_build $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories." - -# crunchstat -cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat" -PKG_VERSION=$(version_from_git) -go get "git.curoverse.com/arvados.git/services/crunchstat" -cd $WORKSPACE/packages/$TARGET -fpm_build $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs" +package_go_binary services/keepstore keepstore \ + "Keep storage daemon, accessible to clients on the LAN" +package_go_binary services/keepproxy keepproxy \ + "Make a Keep cluster accessible to clients that are not on the LAN" +package_go_binary services/keep-web keep-web \ + "Static web hosting service for user data stored in Arvados Keep" +package_go_binary services/datamanager arvados-data-manager \ + "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed" +package_go_binary services/arv-git-httpd arvados-git-httpd \ + "Provide authenticated http access to Arvados-hosted git repositories" +package_go_binary services/crunchstat crunchstat \ + "Gather cpu/memory/network statistics of running Crunch jobs" +package_go_binary tools/keep-rsync keep-rsync \ + "Copy all data from one set of Keep servers to another" # The Python SDK # Please resist the temptation to add --no-python-fix-name to the fpm call here @@ -407,10 +358,50 @@ LIBCLOUD_DIR=$(mktemp -d) fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud rm -rf $LIBCLOUD_DIR -# A few dependencies +# Python 2 dependencies +declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps) +# Add --no-use-wheel if this pip knows it. +pip wheel --help >/dev/null 2>&1 +case "$?" in + 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;; + 2) ;; + *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;; +esac + for deppkg in "${PYTHON_BACKPORTS[@]}"; do outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") - fpm_build "$deppkg" "$outname" + case "$deppkg" in + httplib2|google-api-python-client) + # Work around 0640 permissions on some package files. + # See #7591 and #7991. + pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && ( + set -e + cd "$pyfpm_workdir" + pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg" + tar -xf "$deppkg"-*.tar* + cd "$deppkg"-*/ + "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build + chmod -R go+rX . + set +e + # --iteration 2 provides an upgrade for previously built + # buggy packages. + fpm_build . "$outname" "" python "" --iteration 2 + # The upload step uses the package timestamp to determine + # whether it's new. --no-clobber plays nice with that. + mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET" + ) + if [ 0 != "$?" ]; then + echo "ERROR: $deppkg build process failed" + EXITCODE=1 + fi + if [ -n "$pyfpm_workdir" ]; then + rm -rf "$pyfpm_workdir" + fi + ;; + *) + fpm_build "$deppkg" "$outname" + ;; + esac done # Python 3 dependencies @@ -449,7 +440,7 @@ cd $WORKSPACE/packages/$TARGET # This is the complete package with vendor/bundle included. # It's big, so we do not build it by default. if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then - declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$API_VERSION")" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-api/current/agpl-3.0.txt") debug_echo -e "\n${COMMAND_ARR[@]}\n" @@ -459,7 +450,7 @@ if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then fi # Build the 'bare' package without vendor/bundle. -declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh") +declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$API_VERSION")" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-api/current/agpl-3.0.txt") debug_echo -e "\n${COMMAND_ARR[@]}\n" @@ -509,7 +500,7 @@ cd $WORKSPACE/packages/$TARGET # It's big, so we do not build it by default. if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then - declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") + declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$WORKBENCH_VERSION")" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-workbench/current/agpl-3.0.txt") debug_echo -e "\n${COMMAND_ARR[@]}\n" @@ -520,7 +511,7 @@ fi # Build the 'bare' package without vendor/bundle. -declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh") +declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege " "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$WORKBENCH_VERSION")" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-workbench/current/agpl-3.0.txt") debug_echo -e "\n${COMMAND_ARR[@]}\n"