PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py \
- pykka requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse \
+ 'pycurl<7.21.5')
PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
;;
debian8)
PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py \
- pykka requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse \
+ 'pycurl<7.21.5')
PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
;;
ubuntu1204)
PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py \
- pykka requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl llfuse)
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname llfuse \
+ 'pycurl<7.21.5')
PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
;;
ubuntu1404)
PYTHON3_PKG_PREFIX=python3
PYTHON_BACKPORTS=(pyasn1==0.1.7 pyvcf pyasn1-modules==0.0.5 llfuse ciso8601 \
google-api-python-client six uritemplate oauth2client httplib2 \
- rsa pycurl backports.ssl_match_hostname)
+ rsa 'pycurl<7.21.5' backports.ssl_match_hostname)
PYTHON3_BACKPORTS=(docker-py requests websocket-client)
;;
centos6)
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
PYTHON_BACKPORTS=(python-gflags pyvcf google-api-python-client \
oauth2client pyasn1==0.1.7 pyasn1-modules==0.0.5 \
- rsa uritemplate httplib2 ws4py \
- pykka requests six pyexecjs jsonschema \
- ciso8601 pycrypto backports.ssl_match_hostname pycurl
- python-daemon lockfile llfuse)
+ rsa uritemplate httplib2 ws4py pykka six pyexecjs jsonschema \
+ ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \
+ python-daemon lockfile llfuse 'pbr<1.0')
PYTHON3_BACKPORTS=(docker-py six requests websocket-client)
export PYCURL_SSL_LIBRARY=nss
;;
perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
fpm_build install/lib/=/usr/share libarvados-perl \
- "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man && \
- mv libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
+ "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \
+ "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \
+ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
# Ruby gems
debug_echo -e "\nRuby gems\n"
cd "$WORKSPACE/sdk/python"
handle_python_package
+cd "$WORKSPACE/sdk/cwl"
+handle_python_package
+
cd "$WORKSPACE/services/fuse"
handle_python_package
COMMIT_HASH=$(format_last_commit_here "%H")
SRC_BUILD_DIR=$(mktemp -d)
+ # mktemp creates the directory with 0700 permissions by default
+ chmod 755 $SRC_BUILD_DIR
git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
cd "$SRC_BUILD_DIR"
cd $WORKSPACE/packages/$TARGET
fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
- rm -r "$SRC_BUILD_DIR"
+ rm -rf "$SRC_BUILD_DIR"
)
-# Keep
+# Go binaries
export GOPATH=$(mktemp -d)
-mkdir -p "$GOPATH/src/git.curoverse.com"
-ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
-
-# keepstore
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepstore"
-PKG_VERSION=$(version_from_git)
-go get "git.curoverse.com/arvados.git/services/keepstore"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/keepstore=/usr/bin/keepstore keepstore 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepstore is the Keep storage daemon, accessible to clients on the LAN"
-
-# Get GO SDK version
-cd "$GOPATH/src/git.curoverse.com/arvados.git/sdk/go"
-GO_SDK_VERSION=$(version_from_git)
-GO_SDK_TIMESTAMP=$(timestamp_from_git)
-
-# keepproxy
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/keepproxy"
-KEEPPROXY_VERSION=$(version_from_git)
-KEEPPROXY_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$KEEPPROXY_TIMESTAMP" ]]; then
- PKG_VERSION=$GO_SDK_VERSION
-else
- PKG_VERSION=$KEEPPROXY_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/keepproxy"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/keepproxy=/usr/bin/keepproxy keepproxy 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Keepproxy makes a Keep cluster accessible to clients that are not on the LAN"
-
-# datamanager
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/datamanager"
-DATAMANAGER_VERSION=$(version_from_git)
-DATAMANAGER_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$DATAMANAGER_TIMESTAMP" ]]; then
- PKG_VERSION=$GO_SDK_VERSION
-else
- PKG_VERSION=$DATAMANAGER_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/datamanager"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/datamanager=/usr/bin/arvados-data-manager arvados-data-manager 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Datamanager ensures block replication levels, reports on disk usage and determines which blocks should be deleted when space is needed."
-
-# arv-git-httpd
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/arv-git-httpd"
-ARVGITHTTPD_VERSION=$(version_from_git)
-ARVGITHTTPD_TIMESTAMP=$(timestamp_from_git)
-
-if [[ "$GO_SDK_TIMESTAMP" -gt "$ARVGITHTTPD_TIMESTAMP" ]]; then
- PKG_VERSION=$GO_SDK_VERSION
-else
- PKG_VERSION=$ARVGITHTTPD_VERSION
-fi
-
-go get "git.curoverse.com/arvados.git/services/arv-git-httpd"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/arv-git-httpd=/usr/bin/arvados-git-httpd arvados-git-httpd 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Provides authenticated http access to Arvados-hosted git repositories."
-
-# crunchstat
-cd "$GOPATH/src/git.curoverse.com/arvados.git/services/crunchstat"
-PKG_VERSION=$(version_from_git)
-go get "git.curoverse.com/arvados.git/services/crunchstat"
-cd $WORKSPACE/packages/$TARGET
-fpm_build $GOPATH/bin/crunchstat=/usr/bin/crunchstat crunchstat 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=Crunchstat gathers cpu/memory/network statistics of running Crunch jobs"
+package_go_binary services/keepstore keepstore \
+ "Keep storage daemon, accessible to clients on the LAN"
+package_go_binary services/keepproxy keepproxy \
+ "Make a Keep cluster accessible to clients that are not on the LAN"
+package_go_binary services/keep-web keep-web \
+ "Static web hosting service for user data stored in Arvados Keep"
+package_go_binary services/datamanager arvados-data-manager \
+ "Ensure block replication levels, report disk usage, and determine which blocks should be deleted when space is needed"
+package_go_binary services/arv-git-httpd arvados-git-httpd \
+ "Provide authenticated http access to Arvados-hosted git repositories"
+package_go_binary services/crunchstat crunchstat \
+ "Gather cpu/memory/network statistics of running Crunch jobs"
+package_go_binary tools/keep-rsync keep-rsync \
+ "Copy all data from one set of Keep servers to another"
# The Python SDK
# Please resist the temptation to add --no-python-fix-name to the fpm call here
fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
rm -rf $LIBCLOUD_DIR
-# A few dependencies
+# Python 2 dependencies
+declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps)
+# Add --no-use-wheel if this pip knows it.
+pip wheel --help >/dev/null 2>&1
+case "$?" in
+ 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;;
+ 2) ;;
+ *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;;
+esac
+
for deppkg in "${PYTHON_BACKPORTS[@]}"; do
outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/")
- fpm_build "$deppkg" "$outname"
+ case "$deppkg" in
+ httplib2|google-api-python-client)
+ # Work around 0640 permissions on some package files.
+ # See #7591 and #7991.
+ pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && (
+ set -e
+ cd "$pyfpm_workdir"
+ pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg"
+ tar -xf "$deppkg"-*.tar*
+ cd "$deppkg"-*/
+ "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build
+ chmod -R go+rX .
+ set +e
+ # --iteration 2 provides an upgrade for previously built
+ # buggy packages.
+ fpm_build . "$outname" "" python "" --iteration 2
+ # The upload step uses the package timestamp to determine
+ # whether it's new. --no-clobber plays nice with that.
+ mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET"
+ )
+ if [ 0 != "$?" ]; then
+ echo "ERROR: $deppkg build process failed"
+ EXITCODE=1
+ fi
+ if [ -n "$pyfpm_workdir" ]; then
+ rm -rf "$pyfpm_workdir"
+ fi
+ ;;
+ *)
+ fpm_build "$deppkg" "$outname"
+ ;;
+ esac
done
# Python 3 dependencies
# This is the complete package with vendor/bundle included.
# It's big, so we do not build it by default.
if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$API_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$API_VERSION")" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-api/current/agpl-3.0.txt")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
fi
# Build the 'bare' package without vendor/bundle.
-declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh")
+declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados API server - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$API_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$API_VERSION")" "-x" "var/www/arvados-api/current/tmp" "-x" "var/www/arvados-api/current/log" "-x" "var/www/arvados-api/current/vendor/bundle" "-x" "var/www/arvados-api/current/vendor/cache/*" "-x" "var/www/arvados-api/current/coverage" "-x" "var/www/arvados-api/current/Capfile*" "-x" "var/www/arvados-api/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/postinst.sh" "$WORKSPACE/services/api/=/var/www/arvados-api/current" "$RUN_BUILD_PACKAGES_PATH/arvados-api-server-extras/arvados-api-server-upgrade.sh=/usr/local/bin/arvados-api-server-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-api/current/agpl-3.0.txt")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
# It's big, so we do not build it by default.
if [[ "$BUILD_BUNDLE_PACKAGES" != 0 ]]; then
- declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
+ declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}-with-bundle" "-v" "$WORKBENCH_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$WORKBENCH_VERSION")" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-workbench/current/agpl-3.0.txt")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
# Build the 'bare' package without vendor/bundle.
-declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh")
+declare -a COMMAND_ARR=("fpm" "--maintainer=Ward Vandewege <ward@curoverse.com>" "--vendor='Curoverse, Inc.'" "--url='https://arvados.org'" "--description='Arvados Workbench - Arvados is a free and open source platform for big data science.'" "--license='GNU Affero General Public License, version 3.0'" "-s" "dir" "-t" "$FORMAT" "-n" "${PACKAGE_NAME}" "-v" "$WORKBENCH_VERSION" "--iteration" "$(default_iteration "$PACKAGE_NAME" "$WORKBENCH_VERSION")" "-x" "var/www/arvados-workbench/current/log" "-x" "var/www/arvados-workbench/current/vendor/bundle" "-x" "var/www/arvados-workbench/current/vendor/cache/*" "-x" "var/www/arvados-workbench/current/coverage" "-x" "var/www/arvados-workbench/current/Capfile*" "-x" "var/www/arvados-workbench/current/config/deploy*" "--after-install=$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/postinst.sh" "$WORKSPACE/apps/workbench/=/var/www/arvados-workbench/current" "$RUN_BUILD_PACKAGES_PATH/arvados-workbench-extras/arvados-workbench-upgrade.sh=/usr/local/bin/arvados-workbench-upgrade.sh" "$WORKSPACE/agpl-3.0.txt=/var/www/arvados-workbench/current/agpl-3.0.txt")
debug_echo -e "\n${COMMAND_ARR[@]}\n"
projects / arvados-dev.git / blobdiff