X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/53843c61d600a0911e9f4f512b0d5c34b18f9d33..b7c46875f592b2f959f504e7f583979a594d528f:/build/run-build-packages.sh diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index b800d43e08..5aa0b7e6f8 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: AGPL-3.0 -. `dirname "$(readlink -f "$0")"`/run-library.sh -. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh +. `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1 +. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1 read -rd "\000" helpmessage < - Distribution to build packages for (default: debian8) + Distribution to build packages for (default: debian10) --only-build Build only a specific package (or $ONLY_BUILD from environment) +--force-build + Build even if the package exists upstream or if it has already been + built locally --command Build command to execute (defaults to the run command defined in the Docker image) @@ -35,18 +38,19 @@ EOF # set to --no-cache-dir to disable pip caching CACHE_FLAG= -MAINTAINER="Ward Vandewege " -VENDOR="Veritas Genetics, Inc." +MAINTAINER="Arvados Package Maintainers " +VENDOR="The Arvados Project" # End of user configuration DEBUG=${ARVADOS_DEBUG:-0} +FORCE_BUILD=${FORCE_BUILD:-0} EXITCODE=0 -TARGET=debian8 +TARGET=debian10 COMMAND= PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,build-bundle-packages,debug,target:,only-build: \ + help,build-bundle-packages,debug,target:,only-build:,force-build \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -66,6 +70,9 @@ while [ $# -gt 0 ]; do --only-build) ONLY_BUILD="$2"; shift ;; + --force-build) + FORCE_BUILD=1 + ;; --debug) DEBUG=1 ;; @@ -127,7 +134,7 @@ case "$TARGET" in PYTHON2_INSTALL_LIB=lib/python$PYTHON2_VERSION/site-packages PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n') PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE - PYTHON3_PREFIX=/opt/rh/rh-python35/root/usr + PYTHON3_PREFIX=/opt/rh/rh-python36/root/usr PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages export PYCURL_SSL_LIBRARY=nss ;; @@ -157,14 +164,6 @@ if [[ "$?" != 0 ]]; then exit 1 fi -PYTHON2_FPM_INSTALLER=(--python-easyinstall "$(find_python_program easy_install-$PYTHON2_VERSION easy_install)") -install3=$(find_python_program easy_install-$PYTHON3_VERSION easy_install3 pip-$PYTHON3_VERSION pip3) -if [[ $install3 =~ easy_ ]]; then - PYTHON3_FPM_INSTALLER=(--python-easyinstall "$install3") -else - PYTHON3_FPM_INSTALLER=(--python-pip "$install3") -fi - RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`" RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then @@ -224,7 +223,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \ make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \ - fpm_build install/lib/=/usr/share libarvados-perl \ + fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \ dir "$(version_from_git)" install/man/=/usr/share/man \ "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \ mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/" @@ -272,7 +271,7 @@ debug_echo -e "\nPython packages\n" cd "$SRC_BUILD_DIR" PKG_VERSION=$(version_from_git) cd $WORKSPACE/packages/$TARGET - fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" + fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all" rm -rf "$SRC_BUILD_DIR" fi @@ -281,7 +280,6 @@ debug_echo -e "\nPython packages\n" # Go binaries cd $WORKSPACE/packages/$TARGET export GOPATH=$(mktemp -d) -go get github.com/kardianos/govendor package_go_binary cmd/arvados-client arvados-client \ "Arvados command line tool (beta)" package_go_binary cmd/arvados-server arvados-server \ @@ -290,15 +288,13 @@ package_go_binary cmd/arvados-server arvados-controller \ "Arvados cluster controller daemon" package_go_binary cmd/arvados-server arvados-dispatch-cloud \ "Arvados cluster cloud dispatch" -package_go_binary sdk/go/crunchrunner crunchrunner \ - "Crunchrunner executes a command inside a container and uploads the output" package_go_binary services/arv-git-httpd arvados-git-httpd \ "Provide authenticated http access to Arvados-hosted git repositories" package_go_binary services/crunch-dispatch-local crunch-dispatch-local \ "Dispatch Crunch containers on the local system" package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \ "Dispatch Crunch containers to a SLURM cluster" -package_go_binary services/crunch-run crunch-run \ +package_go_binary cmd/arvados-server crunch-run \ "Supervise a single Crunch container" package_go_binary services/crunchstat crunchstat \ "Gather cpu/memory/network statistics of running Crunch jobs" @@ -312,7 +308,7 @@ package_go_binary services/keepstore keepstore \ "Keep storage daemon, accessible to clients on the LAN" package_go_binary services/keep-web keep-web \ "Static web hosting service for user data stored in Arvados Keep" -package_go_binary services/ws arvados-ws \ +package_go_binary cmd/arvados-server arvados-ws \ "Arvados Websocket server" package_go_binary tools/sync-groups arvados-sync-groups \ "Synchronize remote groups into Arvados from an external source" @@ -322,13 +318,17 @@ package_go_binary tools/keep-rsync keep-rsync \ "Copy all data from one set of Keep servers to another" package_go_binary tools/keep-exercise keep-exercise \ "Performance testing tool for Arvados Keep" +package_go_so lib/pam pam_arvados.so libpam-arvados-go \ + "Arvados PAM authentication module" -# The Python SDK +# The Python SDK - Should be built first because it's needed by others fpm_build_virtualenv "arvados-python-client" "sdk/python" + +# The Python SDK - Python3 package fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3" -# Arvados cwl runner -fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" +# Arvados cwl runner - Only supports Python3 now +fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3" # The PAM module fpm_build_virtualenv "libpam-arvados" "sdk/pam" @@ -336,33 +336,41 @@ fpm_build_virtualenv "libpam-arvados" "sdk/pam" # The FUSE driver fpm_build_virtualenv "arvados-fuse" "services/fuse" +# The FUSE driver - Python3 package +fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3" + # The node manager fpm_build_virtualenv "arvados-node-manager" "services/nodemanager" -# The Docker image cleaner -fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" - # The Arvados crunchstat-summary tool fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" +# The Docker image cleaner +fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3" + # The cwltest package, which lives out of tree cd "$WORKSPACE" if [[ -e "$WORKSPACE/cwltest" ]]; then rm -rf "$WORKSPACE/cwltest" fi git clone https://github.com/common-workflow-language/cwltest.git +# last release to support python 2.7 +(cd cwltest && git checkout 1.0.20190906212748) # signal to our build script that we want a cwltest executable installed in /usr/bin/ mkdir cwltest/bin && touch cwltest/bin/cwltest fpm_build_virtualenv "cwltest" "cwltest" rm -rf "$WORKSPACE/cwltest" +calculate_go_package_version arvados_server_version cmd/arvados-server +arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") + # Build the API server package test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" if [[ "$?" == "0" ]]; then handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ --description="Arvados API server - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" fi # Build the workbench server package @@ -370,6 +378,22 @@ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" if [[ "$?" == "0" ]] ; then ( set -e + + # The workbench package has a build-time dependency on the arvados-server + # package for config manipulation, so install it first. + cd $WORKSPACE/cmd/arvados-server + get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go + + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}" + if [[ ! -e ${arvados_server_pkg_path} ]]; then + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}" + fi + if [[ "$FORMAT" == "deb" ]]; then + dpkg -i ${arvados_server_pkg_path} + else + rpm -i ${arvados_server_pkg_path} + fi + cd "$WORKSPACE/apps/workbench" # We need to bundle to be ready even when we build a package without vendor directory @@ -382,17 +406,17 @@ if [[ "$?" == "0" ]] ; then rm -rf tmp mkdir tmp - # Set up application.yml and production.rb so that asset precompilation works - \cp config/application.yml.example config/application.yml -f - \cp config/environments/production.rb.example config/environments/production.rb -f - sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml - sed -i 's/keep_web_url: false/keep_web_url: exampledotcom/' config/application.yml + # Set up an appropriate config.yml + arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x + mkdir -p /etc/arvados/ + mv /tmp/x /etc/arvados/config.yml + perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml - RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >/dev/null - RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null + RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >"$STDOUT_IF_DEBUG" + RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >"$STDOUT_IF_DEBUG" # Remove generated configuration files so they don't go in the package. - rm config/application.yml config/environments/production.rb + rm -rf /etc/arvados/ ) if [[ "$?" != "0" ]]; then @@ -402,7 +426,7 @@ if [[ "$?" == "0" ]] ; then handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" fi fi