X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c53ad9bc61054880ae6d68db8b868556c7278a78..10a5441e60ef81e9f875e4b49f987dba14622bf2:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index 7ce4323e2c..e8cd721ae2 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -110,9 +110,6 @@ case "$TARGET" in debian9) FORMAT=deb ;; - ubuntu1204) - FORMAT=deb - ;; ubuntu1404) FORMAT=deb ;; @@ -288,59 +285,12 @@ handle_python_package fi ) -# On older platforms we need to publish a backport of libfuse >=2.9.2, -# and we need to build and install it here in order to even build an -# llfuse package. -cd $WORKSPACE/packages/$TARGET -if [[ $TARGET =~ ubuntu1204 ]]; then - # port libfuse 2.9.2 to Ubuntu 12.04 - LIBFUSE_DIR=$(mktemp -d) - ( - cd $LIBFUSE_DIR - # download fuse 2.9.2 ubuntu 14.04 source package - file="fuse_2.9.2.orig.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - file="fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - file="fuse_2.9.2-4ubuntu4.14.04.1.dsc" && curl -L -o "${file}" "http://archive.ubuntu.com/ubuntu/pool/main/f/fuse/${file}" - - # install dpkg-source and dpkg-buildpackage commands - apt-get install -y --no-install-recommends dpkg-dev - - # extract source and apply patches - dpkg-source -x fuse_2.9.2-4ubuntu4.14.04.1.dsc - rm -f fuse_2.9.2.orig.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.debian.tar.xz fuse_2.9.2-4ubuntu4.14.04.1.dsc - - # add new version to changelog - cd fuse-2.9.2 - ( - echo "fuse (2.9.2-5) precise; urgency=low" - echo - echo " * Backported from trusty-security to precise" - echo - echo " -- Joshua Randall Thu, 4 Feb 2016 11:31:00 -0000" - echo - cat debian/changelog - ) > debian/changelog.new - mv debian/changelog.new debian/changelog - - # install build-deps and build - apt-get install -y --no-install-recommends debhelper dh-autoreconf libselinux-dev - dpkg-buildpackage -rfakeroot -b - ) - fpm_build "$LIBFUSE_DIR/fuse_2.9.2-5_amd64.deb" fuse "Ubuntu Developers" deb "2.9.2" --iteration 5 - fpm_build "$LIBFUSE_DIR/libfuse2_2.9.2-5_amd64.deb" libfuse2 "Ubuntu Developers" deb "2.9.2" --iteration 5 - fpm_build "$LIBFUSE_DIR/libfuse-dev_2.9.2-5_amd64.deb" libfuse-dev "Ubuntu Developers" deb "2.9.2" --iteration 5 - dpkg -i \ - "$WORKSPACE/packages/$TARGET/fuse_2.9.2-5_amd64.deb" \ - "$WORKSPACE/packages/$TARGET/libfuse2_2.9.2-5_amd64.deb" \ - "$WORKSPACE/packages/$TARGET/libfuse-dev_2.9.2-5_amd64.deb" - apt-get -y --no-install-recommends -f install - rm -rf $LIBFUSE_DIR -fi - # Go binaries cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) -go get -v github.com/kardianos/govendor +go get github.com/kardianos/govendor +package_go_binary cmd/arvados-client arvados-client \ + "Arvados command line tool (beta)" package_go_binary sdk/go/crunchrunner crunchrunner \ "Crunchrunner executes a command inside a container and uploads the output" package_go_binary services/arv-git-httpd arvados-git-httpd \ @@ -365,7 +315,7 @@ package_go_binary services/keep-web keep-web \ "Static web hosting service for user data stored in Arvados Keep" package_go_binary services/ws arvados-ws \ "Arvados Websocket server" -package_go_binary tools/arv-sync-groups arv-sync-groups \ +package_go_binary tools/sync-groups arvados-sync-groups \ "Synchronize remote groups into Arvados from an external source" package_go_binary tools/keep-block-check keep-block-check \ "Verify that all data from one set of Keep servers to another was copied" @@ -383,7 +333,7 @@ package_go_binary tools/keep-exercise keep-exercise \ # 2014-05-15 cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/sdk/python/build" -arvados_python_client_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO) +arvados_python_client_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)} test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-python-client "$arvados_python_client_version" python if [[ "$?" == "0" ]]; then fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$arvados_python_client_version" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git @@ -392,11 +342,17 @@ fi # cwl-runner cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/sdk/cwl/build" -arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO) -arvados_cwl_runner_iteration=3 +arvados_cwl_runner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)} +declare -a iterargs=() +if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then + arvados_cwl_runner_iteration=3 + iterargs+=(--iteration $arvados_cwl_runner_iteration) +else + arvados_cwl_runner_iteration= +fi test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration" if [[ "$?" == "0" ]]; then - fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration + fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}" fi # schema_salad. This is a python dependency of arvados-cwl-runner, @@ -415,16 +371,16 @@ fi # # Ward, 2016-03-17 saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/") -test_package_presence python-schema-salad "$saladversion" python +test_package_presence python-schema-salad "$saladversion" python 2 if [[ "$?" == "0" ]]; then - fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" + fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2 fi # And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17 cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/") -test_package_presence python-cwltool "$cwltoolversion" python +test_package_presence python-cwltool "$cwltoolversion" python 2 if [[ "$?" == "0" ]]; then - fpm_build cwltool "" "" python $cwltoolversion + fpm_build cwltool "" "" python $cwltoolversion --iteration 2 fi # The PAM module @@ -443,7 +399,7 @@ fi # not omit the python- prefix first. cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/services/fuse/build" -arvados_fuse_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO) +arvados_fuse_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)} test_package_presence "${PYTHON2_PKG_PREFIX}-arvados-fuse" "$arvados_fuse_version" python if [[ "$?" == "0" ]]; then fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$arvados_fuse_version" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools" @@ -452,7 +408,7 @@ fi # The node manager cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/services/nodemanager/build" -nodemanager_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO) +nodemanager_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)} test_package_presence arvados-node-manager "$nodemanager_version" python if [[ "$?" == "0" ]]; then fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$nodemanager_version" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools" @@ -461,38 +417,25 @@ fi # The Docker image cleaner cd $WORKSPACE/packages/$TARGET rm -rf "$WORKSPACE/services/dockercleaner/build" -dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO) -dockercleaner_iteration=3 -test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration" +dockercleaner_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)} +iteration="${ARVADOS_BUILDING_ITERATION:-3}" +test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration" if [[ "$?" == "0" ]]; then - fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration" + fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration" fi # The Arvados crunchstat-summary tool cd $WORKSPACE/packages/$TARGET -crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO) -test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python +crunchstat_summary_version=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)} +iteration="${ARVADOS_BUILDING_ITERATION:-2}" +test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration" if [[ "$?" == "0" ]]; then rm -rf "$WORKSPACE/tools/crunchstat-summary/build" - fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" + fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration" fi -if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then - # Forked libcloud - LIBCLOUD_DIR=$(mktemp -d) - ( - cd $LIBCLOUD_DIR - git clone $DASHQ_UNLESS_DEBUG https://github.com/curoverse/libcloud.git . - git checkout $DASHQ_UNLESS_DEBUG apache-libcloud-$LIBCLOUD_PIN - # libcloud is absurdly noisy without -q, so force -q here - OLD_DASHQ_UNLESS_DEBUG=$DASHQ_UNLESS_DEBUG - DASHQ_UNLESS_DEBUG=-q - handle_python_package - DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG - ) - fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud - rm -rf $LIBCLOUD_DIR -fi +## if libcloud becomes our own fork see +## https://dev.arvados.org/issues/12268#note-27 # Python 2 dependencies declare -a PIP_DOWNLOAD_SWITCHES=(--no-deps) @@ -612,7 +555,7 @@ if [[ "$?" == "0" ]] ; then # We need to bundle to be ready even when we build a package without vendor directory # because asset compilation requires it. - bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG" + bundle install --system >"$STDOUT_IF_DEBUG" # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, # and we want that in the package, so it's easier to not exclude the tmp directory