X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0b3b1e9d91f06966cbeecd3d4a11e1fb7e8d8434..43ad590772de48fbc3a6a45654445bab79a0bdc1:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 08b7e21368..ef1df03d5a 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -1,7 +1,10 @@ #!/bin/bash +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 . `dirname "$(readlink -f "$0")"`/run-library.sh -. `dirname "$(readlink -f "$0")"`/libcloud-pin +. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh read -rd "\000" helpmessage < - Distribution to build packages for (default: debian7) + Distribution to build packages for (default: debian8) --only-build Build only a specific package (or $ONLY_BUILD from environment) --command @@ -27,9 +30,19 @@ WORKSPACE=path Path to the Arvados source tree to build packages from EOF -EXITCODE=0 +# Begin of user configuration + +# set to --no-cache-dir to disable pip caching +CACHE_FLAG= + +MAINTAINER="Ward Vandewege " +VENDOR="Veritas Genetics, Inc." + +# End of user configuration + DEBUG=${ARVADOS_DEBUG:-0} -TARGET=debian7 +EXITCODE=0 +TARGET=debian8 COMMAND= PARSEDOPTS=$(getopt --name "$0" --longoptions \ @@ -101,87 +114,21 @@ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages ## End Debian Python defaults. case "$TARGET" in - debian7) - FORMAT=deb - PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ - oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ - 'pycurl<7.21.5' contextlib2 pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict - cachecontrol requests 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0) - ;; - debian8) - FORMAT=deb - PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ - oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ - 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict - cachecontrol 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0) - ;; - ubuntu1204) + debian*) FORMAT=deb - PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ - oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six \ - ciso8601 pycrypto backports.ssl_match_hostname llfuse==0.41.1 \ - contextlib2 'pycurl<7.21.5' pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro isodate ruamel.ordereddict - cachecontrol requests 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0) ;; - ubuntu1404) + ubuntu*) FORMAT=deb - PYTHON_BACKPORTS=(pyasn1==0.1.7 pyasn1-modules==0.0.5 llfuse==0.41.1 ciso8601 \ - google-api-python-client==1.4.2 six uritemplate oauth2client==1.5.2 httplib2 \ - rsa 'pycurl<7.21.5' backports.ssl_match_hostname pyyaml 'rdflib>=4.2.0' \ - shellescape mistune typing avro ruamel.ordereddict - cachecontrol 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 requests websocket-client==0.37.0) - ;; - centos6) - FORMAT=rpm - PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') - PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE - PYTHON2_PREFIX=/opt/rh/python27/root/usr - PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages - PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') - PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE - PYTHON3_PREFIX=/opt/rh/python33/root/usr - PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages - PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ - oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka six \ - ciso8601 pycrypto backports.ssl_match_hostname 'pycurl<7.21.5' \ - python-daemon llfuse==0.41.1 'pbr<1.0' pyyaml \ - 'rdflib>=4.2.0' shellescape mistune typing avro requests \ - isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \ - ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0) - export PYCURL_SSL_LIBRARY=nss ;; - centos7) + centos*) FORMAT=rpm PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n') PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE - PYTHON3_PREFIX=/opt/rh/python33/root/usr + PYTHON3_PREFIX=/opt/rh/rh-python35/root/usr PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages - PYTHON_BACKPORTS=(python-gflags==2.0 google-api-python-client==1.4.2 \ - oauth2client==1.5.2 pyasn1==0.1.7 pyasn1-modules==0.0.5 \ - rsa uritemplate httplib2 ws4py pykka \ - ciso8601 pycrypto 'pycurl<7.21.5' \ - python-daemon==2.1.1 llfuse==0.41.1 'pbr<1.0' pyyaml \ - 'rdflib>=4.2.0' shellescape mistune typing avro \ - isodate pyparsing sparqlwrapper html5lib==0.9999999 keepalive \ - ruamel.ordereddict cachecontrol 'pathlib2>=2.1.0') - PYTHON3_BACKPORTS=(docker-py==1.7.2 six requests websocket-client==0.37.0) export PYCURL_SSL_LIBRARY=nss ;; *) @@ -210,8 +157,13 @@ if [[ "$?" != 0 ]]; then exit 1 fi -EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "") -EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3) +PYTHON2_FPM_INSTALLER=(--python-easyinstall "$(find_python_program easy_install-$PYTHON2_VERSION easy_install)") +install3=$(find_python_program easy_install-$PYTHON3_VERSION easy_install3 pip-$PYTHON3_VERSION pip3) +if [[ $install3 =~ easy_ ]]; then + PYTHON3_FPM_INSTALLER=(--python-easyinstall "$install3") +else + PYTHON3_FPM_INSTALLER=(--python-pip "$install3") +fi RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`" RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized @@ -254,21 +206,29 @@ fi debug_echo -e "\nPerl packages\n" if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then -cd "$WORKSPACE/sdk/perl" + cd "$WORKSPACE/sdk/perl" + libarvados_perl_version="$(version_from_git)" -if [[ -e Makefile ]]; then - make realclean >"$STDOUT_IF_DEBUG" -fi -find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ - -delete -rm -rf install - -perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ - make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ - fpm_build install/lib/=/usr/share libarvados-perl \ - "Curoverse, Inc." dir "$(version_from_git)" install/man/=/usr/share/man \ - "$WORKSPACE/LICENSE-2.0.txt=/usr/share/doc/libarvados-perl/LICENSE-2.0.txt" && \ - mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" + cd $WORKSPACE/packages/$TARGET + test_package_presence libarvados-perl "$libarvados_perl_version" + + if [[ "$?" == "0" ]]; then + cd "$WORKSPACE/sdk/perl" + + if [[ -e Makefile ]]; then + make realclean >"$STDOUT_IF_DEBUG" + fi + find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \ + -delete + rm -rf install + + perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ + make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ + fpm_build install/lib/=/usr/share libarvados-perl \ + dir "$(version_from_git)" install/man/=/usr/share/man \ + "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \ + mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" + fi fi # Ruby gems @@ -305,113 +265,46 @@ handle_python_package # arvados-src ( - set -e - cd "$WORKSPACE" COMMIT_HASH=$(format_last_commit_here "%H") + arvados_src_version="$(version_from_git)" - SRC_BUILD_DIR=$(mktemp -d) - # mktemp creates the directory with 0700 permissions by default - chmod 755 $SRC_BUILD_DIR - git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" - cd "$SRC_BUILD_DIR" - - # go into detached-head state - git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" - echo "$COMMIT_HASH" >git-commit.version - - cd "$SRC_BUILD_DIR" - PKG_VERSION=$(version_from_git) cd $WORKSPACE/packages/$TARGET - fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'Curoverse, Inc.' 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" - - rm -rf "$SRC_BUILD_DIR" + test_package_presence arvados-src $arvados_src_version src "" + + if [[ "$?" == "0" ]]; then + cd "$WORKSPACE" + SRC_BUILD_DIR=$(mktemp -d) + # mktemp creates the directory with 0700 permissions by default + chmod 755 $SRC_BUILD_DIR + git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR" + cd "$SRC_BUILD_DIR" + + # go into detached-head state + git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH" + echo "$COMMIT_HASH" >git-commit.version + + cd "$SRC_BUILD_DIR" + PKG_VERSION=$(version_from_git) + cd $WORKSPACE/packages/$TARGET + fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" + + rm -rf "$SRC_BUILD_DIR" + fi ) -# On older platforms we need to publish a backport of libfuse >=2.9.2, -# and we need to build and install it here in order to even build an -# llfuse package. -cd $WORKSPACE/packages/$TARGET -if [[ $TARGET =~ ubuntu1204 ]]; then - # port libfuse 2.9.2 to Ubuntu 12.04 - LIBFUSE_DIR=$(mktemp -d) - ( - cd $LIBFUSE_DIR - # download fuse 2.9.2 ubuntu 14.04 source package - file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - - # install dpkg-source and dpkg-buildpackage commands - apt-get install -y --no-install-recommends dpkg-dev - - # extract source and apply patches - dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc - rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc - - # add new version to changelog - cd fuse-2.9.2 - ( - echo "fuse (2.9.2-5) precise; urgency=low" - echo - echo " * Backported from trusty-security to precise" - echo - echo " -- Joshua Randall Thu, 4 Feb 2016 11:31:00 -0000" - echo - cat debian/changelog - ) > debian/changelog.new - mv debian/changelog.new debian/changelog - - # install build-deps and build - apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev - dpkg-buildpackage -rfakeroot -b - ) - fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5 - fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5 - fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5 - dpkg -i \ - "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \ - "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \ - "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb" - apt-get -y --no-install-recommends -f install - rm -rf $LIBFUSE_DIR -elif [[ $TARGET =~ centos6 ]]; then - # port fuse 2.9.2 to centos 6 - # install tools to build rpm from source - yum install -y rpm-build redhat-rpm-config - LIBFUSE_DIR=$(mktemp -d) - ( - cd "$LIBFUSE_DIR" - # download fuse 2.9.2 centos 7 source rpm - file="fuse-2.9.2-6.el7.src.rpm" && curl -L -o "${file}" "http://vault.centos.org/7.2.1511/os/Source/SPackages/${file}" - ( - # modify source rpm spec to remove conflict on filesystem version - mkdir -p /root/rpmbuild/SOURCES - cd /root/rpmbuild/SOURCES - rpm2cpio ${LIBFUSE_DIR}/fuse-2.9.2-6.el7.src.rpm | cpio -i - perl -pi -e 's/Conflicts:\s*filesystem.*//g' fuse.spec - ) - # build rpms from source - rpmbuild -bb /root/rpmbuild/SOURCES/fuse.spec - rm -f fuse-2.9.2-6.el7.src.rpm - # move built RPMs to LIBFUSE_DIR - mv "/root/rpmbuild/RPMS/x86_64/fuse-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ - mv "/root/rpmbuild/RPMS/x86_64/fuse-libs-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ - mv "/root/rpmbuild/RPMS/x86_64/fuse-devel-2.9.2-6.el6.x86_64.rpm" ${LIBFUSE_DIR}/ - rm -rf /root/rpmbuild - ) - fpm_build "$LIBFUSE_DIR/fuse-libs-2.9.2-6.el6.x86_64.rpm" fuse-libs "Centos Developers" rpm "2.9.2" --iteration 5 - fpm_build "$LIBFUSE_DIR/fuse-2.9.2-6.el6.x86_64.rpm" fuse "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends - fpm_build "$LIBFUSE_DIR/fuse-devel-2.9.2-6.el6.x86_64.rpm" fuse-devel "Centos Developers" rpm "2.9.2" --iteration 5 --no-auto-depends - yum install -y \ - "$WORKSPACE/packages/$TARGET/fuse-libs-2.9.2-5.x86_64.rpm" \ - "$WORKSPACE/packages/$TARGET/fuse-2.9.2-5.x86_64.rpm" \ - "$WORKSPACE/packages/$TARGET/fuse-devel-2.9.2-5.x86_64.rpm" -fi - # Go binaries cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) +go get github.com/kardianos/govendor +package_go_binary cmd/arvados-client arvados-client \ + "Arvados command line tool (beta)" +package_go_binary cmd/arvados-server arvados-server \ + "Arvados server daemons" +package_go_binary cmd/arvados-server arvados-controller \ + "Arvados cluster controller daemon" +package_go_binary cmd/arvados-server arvados-dispatch-cloud \ + "Arvados cluster cloud dispatch" package_go_binary sdk/go/crunchrunner crunchrunner \ "Crunchrunner executes a command inside a container and uploads the output" package_go_binary services/arv-git-httpd arvados-git-httpd \ @@ -424,6 +317,8 @@ package_go_binary services/crunch-run crunch-run \ "Supervise a single Crunch container" package_go_binary services/crunchstat crunchstat \ "Gather cpu/memory/network statistics of running Crunch jobs" +package_go_binary services/health arvados-health \ + "Check health of all Arvados cluster services" package_go_binary services/keep-balance keep-balance \ "Rebalance and garbage-collect data blocks stored in Arvados Keep" package_go_binary services/keepproxy keepproxy \ @@ -434,6 +329,8 @@ package_go_binary services/keep-web keep-web \ "Static web hosting service for user data stored in Arvados Keep" package_go_binary services/ws arvados-ws \ "Arvados Websocket server" +package_go_binary tools/sync-groups arvados-sync-groups \ + "Synchronize remote groups into Arvados from an external source" package_go_binary tools/keep-block-check keep-block-check \ "Verify that all data from one set of Keep servers to another was copied" package_go_binary tools/keep-rsync keep-rsync \ @@ -442,203 +339,77 @@ package_go_binary tools/keep-exercise keep-exercise \ "Performance testing tool for Arvados Keep" # The Python SDK -# Please resist the temptation to add --no-python-fix-name to the fpm call here -# (which would remove the python- prefix from the package name), because this -# package is a dependency of arvados-fuse, and fpm can not omit the python- -# prefix from only one of the dependencies of a package... Maybe I could -# whip up a patch and send it upstream, but that will be for another day. Ward, -# 2014-05-15 -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/sdk/python/build" -fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git - -# cwl-runner -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/sdk/cwl/build" -fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration 3 - -fpm_build lockfile "" "" python 0.12.2 --epoch 1 - -# schema_salad. This is a python dependency of arvados-cwl-runner, -# but we can't use the usual PYTHONPACKAGES way to build this package due to the -# intricacies of how version numbers get generated in setup.py: we need a specific version, -# e.g. 1.7.20160316203940. If we don't explicitly list that version with the -v -# argument to fpm, and instead specify it as schema_salad==1.7.20160316203940, we get -# a package with version 1.7. That's because our gittagger hack is not being -# picked up by self.distribution.get_version(), which is called from -# https://github.com/jordansissel/fpm/blob/master/lib/fpm/package/pyfpm/get_metadata.py -# by means of this command: -# -# python2.7 setup.py --command-packages=pyfpm get_metadata --output=metadata.json -# -# So we build this thing separately. -# -# Ward, 2016-03-17 -saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/") -fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" +fpm_build_virtualenv "arvados-python-client" "sdk/python" +fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" -# And schema_salad now depends on ruamel-yaml, which apparently has a braindead setup.py that requires special arguments to build (otherwise, it aborts with 'error: you have to install with "pip install ."'). Sigh. -# Ward, 2016-05-26 -fpm_build ruamel.yaml "" "" python 0.13.7 --python-setup-py-arguments "--single-version-externally-managed" - -# Dependency of cwltool. Fpm doesn't produce a package with the correct version -# number unless we build it explicitly -fpm_build cwltest "" "" python 1.0.20160907111242 - -# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17 -cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/") -fpm_build cwltool "" "" python $cwltoolversion - -# FPM eats the trailing .0 in the python-rdflib-jsonld package when built with 'rdflib-jsonld>=0.3.0'. Force the version. Ward, 2016-03-25 -fpm_build rdflib-jsonld "" "" python 0.4.0 +# Arvados cwl runner +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" # The PAM module -if [[ $TARGET =~ debian|ubuntu ]]; then - cd $WORKSPACE/packages/$TARGET - rm -rf "$WORKSPACE/sdk/pam/build" - fpm_build $WORKSPACE/sdk/pam libpam-arvados 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/pam/arvados_pam.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=PAM module for authenticating shell logins using Arvados API tokens" --depends libpam-python -fi +fpm_build_virtualenv "libpam-arvados" "sdk/pam" # The FUSE driver -# Please see comment about --no-python-fix-name above; we stay consistent and do -# not omit the python- prefix first. -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/services/fuse/build" -fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools" +fpm_build_virtualenv "arvados-fuse" "services/fuse" # The node manager -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/services/nodemanager/build" -fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools" +fpm_build_virtualenv "arvados-node-manager" "services/nodemanager" # The Docker image cleaner -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/services/dockercleaner/build" -fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration 3 +fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" # The Arvados crunchstat-summary tool -cd $WORKSPACE/packages/$TARGET -rm -rf "$WORKSPACE/tools/crunchstat-summary/build" -fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" - -# Forked libcloud -LIBCLOUD_DIR=$(mktemp -d) -( - cd $LIBCLOUD_DIR - git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git . - git checkout apache-libcloud-$LIBCLOUD_PIN - # libcloud is absurdly noisy without -q, so force -q here - OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG - DASHQ_UNLESS_DEBUG=-q - handle_python_package - DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG -) -fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud -rm -rf $LIBCLOUD_DIR - -# Python 2 dependencies -declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps) -# Add --no-use-wheel if this pip knows it. -pip wheel --help >/dev/null 2>&1 -case "$?" in - 0) PIP_DOWNLOAD_SWITCHES+=(--no-use-wheel) ;; - 2) ;; - *) echo "WARNING: `pip wheel` test returned unknown exit code $?" ;; -esac - -for deppkg in "${PYTHON_BACKPORTS[@]}"; do - outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON2_PKG_PREFIX}-/") - - if [[ -n "$ONLY_BUILD" ]] && [[ "$outname" != "$ONLY_BUILD" ]] ; then - continue - fi - - case "$deppkg" in - httplib2|google-api-python-client) - # Work around 0640 permissions on some package files. - # See #7591 and #7991. - pyfpm_workdir=$(mktemp --tmpdir -d pyfpm-XXXXXX) && ( - set -e - cd "$pyfpm_workdir" - pip install "${PIP_DOWNLOAD_SWITCHES[@]}" --download . "$deppkg" - # Sometimes pip gives us a tarball, sometimes a zip file... - DOWNLOADED=`ls $deppkg-*` - [[ "$DOWNLOADED" =~ ".tar" ]] && tar -xf $DOWNLOADED - [[ "$DOWNLOADED" =~ ".zip" ]] && unzip $DOWNLOADED - cd "$deppkg"-*/ - "python$PYTHON2_VERSION" setup.py $DASHQ_UNLESS_DEBUG egg_info build - chmod -R go+rX . - set +e - fpm_build . "$outname" "" python "" --iteration 3 - # The upload step uses the package timestamp to determine - # whether it's new. --no-clobber plays nice with that. - mv --no-clobber "$outname"*.$FORMAT "$WORKSPACE/packages/$TARGET" - ) - if [ 0 != "$?" ]; then - echo "ERROR: $deppkg build process failed" - EXITCODE=1 - fi - if [ -n "$pyfpm_workdir" ]; then - rm -rf "$pyfpm_workdir" - fi - ;; - *) - fpm_build "$deppkg" "$outname" - ;; - esac -done - -# Python 3 dependencies -for deppkg in "${PYTHON3_BACKPORTS[@]}"; do - outname=$(echo "$deppkg" | sed -e 's/^python-//' -e 's/[<=>].*//' -e 's/_/-/g' -e "s/^/${PYTHON3_PKG_PREFIX}-/") - # The empty string is the vendor argument: these aren't Curoverse software. - fpm_build "$deppkg" "$outname" "" python3 -done +fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" # Build the API server package -handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ - "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados API server - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" - -# Build the workbench server package -if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-workbench" = "$ONLY_BUILD" ]] ; then -( - set -e - cd "$WORKSPACE/apps/workbench" - - # We need to bundle to be ready even when we build a package without vendor directory - # because asset compilation requires it. - bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG" - - # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, - # and we want that in the package, so it's easier to not exclude the tmp directory - # from the package - empty it instead. - rm -rf tmp - mkdir tmp - - # Set up application.yml and production.rb so that asset precompilation works - \cp config/application.yml.example config/application.yml -f - \cp config/environments/production.rb.example config/environments/production.rb -f - sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml - - RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null - - # Remove generated configuration files so they don't go in the package. - rm config/application.yml config/environments/production.rb -) -fi - -if [[ "$?" != "0" ]]; then - echo "ERROR: Asset precompilation failed" - EXITCODE=1 -else - handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ +test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" +if [[ "$?" == "0" ]]; then + handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ - --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ + --description="Arvados API server - Arvados is a free and open source platform for big data science." \ --license="GNU Affero General Public License, version 3.0" fi +# Build the workbench server package +test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" +if [[ "$?" == "0" ]] ; then + ( + set -e + cd "$WORKSPACE/apps/workbench" + + # We need to bundle to be ready even when we build a package without vendor directory + # because asset compilation requires it. + bundle install --system >"$STDOUT_IF_DEBUG" + + # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, + # and we want that in the package, so it's easier to not exclude the tmp directory + # from the package - empty it instead. + rm -rf tmp + mkdir tmp + + # Set up application.yml and production.rb so that asset precompilation works + \cp config/application.yml.example config/application.yml -f + \cp config/environments/production.rb.example config/environments/production.rb -f + sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml + sed -i 's/keep_web_url: false/keep_web_url: exampledotcom/' config/application.yml + + RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >/dev/null + RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null + + # Remove generated configuration files so they don't go in the package. + rm config/application.yml config/environments/production.rb + ) + + if [[ "$?" != "0" ]]; then + echo "ERROR: Asset precompilation failed" + EXITCODE=1 + else + handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ + "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ + --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ + --license="GNU Affero General Public License, version 3.0" + fi +fi + # clean up temporary GOPATH rm -rf "$GOPATH"