build/run-build-packages.sh

   1 #!/bin/bash
   2 # Copyright (C) The Arvados Authors. All rights reserved.
   3 #
   4 # SPDX-License-Identifier: AGPL-3.0
   5
   6 . `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
   7 . `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
   8
   9 read -rd "\000" helpmessage <<EOF
  10 $(basename $0): Build Arvados packages
  11
  12 Syntax:
  13         WORKSPACE=/path/to/arvados $(basename $0) [options]
  14
  15 Options:
  16
  17 --build-bundle-packages  (default: false)
  18     Build api server and workbench packages with vendor/bundle included
  19 --debug
  20     Output debug information (default: false)
  21 --target <target>
  22     Distribution to build packages for (default: debian10)
  23 --only-build <package>
  24     Build only a specific package (or $ONLY_BUILD from environment)
  25 --arch <arch>
  26     Build a specific architecture (or $ARCH from environment, defaults to native architecture)
  27 --force-build
  28     Build even if the package exists upstream or if it has already been
  29     built locally
  30 --command
  31     Build command to execute (defaults to the run command defined in the
  32     Docker image)
  33
  34 WORKSPACE=path         Path to the Arvados source tree to build packages from
  35
  36 EOF
  37
  38 # Begin of user configuration
  39
  40 # set to --no-cache-dir to disable pip caching
  41 CACHE_FLAG=
  42
  43 MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
  44 VENDOR="The Arvados Project"
  45
  46 # End of user configuration
  47
  48 DEBUG=${ARVADOS_DEBUG:-0}
  49 FORCE_BUILD=${FORCE_BUILD:-0}
  50 EXITCODE=0
  51 TARGET=debian10
  52 COMMAND=
  53
  54 PARSEDOPTS=$(getopt --name "$0" --longoptions \
  55     help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
  56     -- "" "$@")
  57 if [ $? -ne 0 ]; then
  58     exit 1
  59 fi
  60
  61 eval set -- "$PARSEDOPTS"
  62 while [ $# -gt 0 ]; do
  63     case "$1" in
  64         --help)
  65             echo >&2 "$helpmessage"
  66             echo >&2
  67             exit 1
  68             ;;
  69         --target)
  70             TARGET="$2"; shift
  71             ;;
  72         --only-build)
  73             ONLY_BUILD="$2"; shift
  74             ;;
  75         --force-build)
  76             FORCE_BUILD=1
  77             ;;
  78         --arch)
  79             ARCH="$2"; shift
  80             ;;
  81         --debug)
  82             DEBUG=1
  83             ;;
  84         --command)
  85             COMMAND="$2"; shift
  86             ;;
  87         --)
  88             if [ $# -gt 1 ]; then
  89                 echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
  90                 exit 1
  91             fi
  92             ;;
  93     esac
  94     shift
  95 done
  96
  97 if [[ "$COMMAND" != "" ]]; then
  98   COMMAND="/usr/local/rvm/bin/rvm-exec default bash /jenkins/$COMMAND --target $TARGET"
  99 fi
 100
 101 STDOUT_IF_DEBUG=/dev/null
 102 STDERR_IF_DEBUG=/dev/null
 103 DASHQ_UNLESS_DEBUG=-q
 104 if [[ "$DEBUG" != 0 ]]; then
 105     STDOUT_IF_DEBUG=/dev/stdout
 106     STDERR_IF_DEBUG=/dev/stderr
 107     DASHQ_UNLESS_DEBUG=
 108 fi
 109
 110 declare -a PYTHON3_BACKPORTS
 111
 112 PYTHON3_VERSION=$(python3 -c 'import sys; print("{v.major}.{v.minor}".format(v=sys.version_info))')
 113
 114 ## These defaults are suitable for any Debian-based distribution.
 115 # You can customize them as needed in distro sections below.
 116 PYTHON3_PACKAGE=python$PYTHON3_VERSION
 117 PYTHON3_PKG_PREFIX=python3
 118 PYTHON3_PREFIX=/usr
 119 PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 120 ## End Debian Python defaults.
 121
 122 case "$TARGET" in
 123     debian*)
 124         FORMAT=deb
 125         ;;
 126     ubuntu*)
 127         FORMAT=deb
 128         ;;
 129     centos*)
 130         FORMAT=rpm
 131         PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
 132         PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
 133         PYTHON3_PREFIX=/usr
 134         PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
 135         export PYCURL_SSL_LIBRARY=nss
 136         ;;
 137     *)
 138         echo -e "$0: Unknown target '$TARGET'.\n" >&2
 139         exit 1
 140         ;;
 141 esac
 142
 143
 144 if ! [[ -n "$WORKSPACE" ]]; then
 145   echo >&2 "$helpmessage"
 146   echo >&2
 147   echo >&2 "Error: WORKSPACE environment variable not set"
 148   echo >&2
 149   exit 1
 150 fi
 151
 152 # Test for fpm
 153 fpm --version >/dev/null 2>&1
 154
 155 if [[ "$?" != 0 ]]; then
 156   echo >&2 "$helpmessage"
 157   echo >&2
 158   echo >&2 "Error: fpm not found"
 159   echo >&2
 160   exit 1
 161 fi
 162
 163 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 164 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
 165 if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
 166   # error; for some reason, the path is not accessible
 167   # to the script (e.g. permissions re-evaled after suid)
 168   exit 1  # fail
 169 fi
 170
 171 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 172 debug_echo "Workspace is $WORKSPACE"
 173
 174 if [[ -f /etc/profile.d/rvm.sh ]]; then
 175     source /etc/profile.d/rvm.sh
 176     GEM="rvm-exec default gem"
 177 else
 178     GEM=gem
 179 fi
 180
 181 # Make all files world-readable -- jenkins runs with umask 027, and has checked
 182 # out our git tree here
 183 chmod o+r "$WORKSPACE" -R
 184
 185 # More cleanup - make sure all executables that we'll package are 755
 186 cd "$WORKSPACE"
 187 find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
 188
 189 # Now fix our umask to something better suited to building and publishing
 190 # gems and packages
 191 umask 0022
 192
 193 debug_echo "umask is" `umask`
 194
 195 if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
 196   mkdir -p $WORKSPACE/packages/$TARGET
 197   chown --reference="$WORKSPACE" "$WORKSPACE/packages/$TARGET"
 198 fi
 199
 200 # Perl packages
 201 debug_echo -e "\nPerl packages\n"
 202
 203 if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
 204   cd "$WORKSPACE/sdk/perl"
 205   libarvados_perl_version="$(version_from_git)"
 206
 207   cd $WORKSPACE/packages/$TARGET
 208   test_package_presence libarvados-perl "$libarvados_perl_version"
 209
 210   if [[ "$?" == "0" ]]; then
 211     cd "$WORKSPACE/sdk/perl"
 212
 213     if [[ -e Makefile ]]; then
 214       make realclean >"$STDOUT_IF_DEBUG"
 215     fi
 216     find -maxdepth 1 \( -name 'MANIFEST*' -or -name "libarvados-perl*.$FORMAT" \) \
 217         -delete
 218     rm -rf install
 219
 220     perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
 221         make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
 222         fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
 223         dir "$(version_from_git)" install/man/=/usr/share/man \
 224         "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
 225         mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
 226   fi
 227 fi
 228
 229 # Ruby gems
 230 debug_echo -e "\nRuby gems\n"
 231
 232 FPM_GEM_PREFIX=$($GEM environment gemdir)
 233
 234 cd "$WORKSPACE/sdk/ruby"
 235 handle_ruby_gem arvados
 236
 237 cd "$WORKSPACE/sdk/cli"
 238 handle_ruby_gem arvados-cli
 239
 240 cd "$WORKSPACE/services/login-sync"
 241 handle_ruby_gem arvados-login-sync
 242
 243 # Python packages
 244 debug_echo -e "\nPython packages\n"
 245
 246 if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-src" == "$ONLY_BUILD" ]] ; then
 247   # arvados-src
 248   (
 249       cd "$WORKSPACE"
 250       COMMIT_HASH=$(format_last_commit_here "%H")
 251       arvados_src_version="$(version_from_git)"
 252
 253       cd $WORKSPACE/packages/$TARGET
 254       test_package_presence arvados-src "$arvados_src_version" src ""
 255
 256       if [[ "$?" == "0" ]]; then
 257         cd "$WORKSPACE"
 258         SRC_BUILD_DIR=$(mktemp -d)
 259         # mktemp creates the directory with 0700 permissions by default
 260         chmod 755 $SRC_BUILD_DIR
 261         git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
 262         cd "$SRC_BUILD_DIR"
 263
 264         # go into detached-head state
 265         git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
 266         echo "$COMMIT_HASH" >git-commit.version
 267
 268         cd $WORKSPACE/packages/$TARGET
 269         fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
 270
 271         rm -rf "$SRC_BUILD_DIR"
 272       fi
 273   )
 274 fi
 275
 276 # Go binaries
 277 cd $WORKSPACE/packages/$TARGET
 278 export GOPATH=$(mktemp -d)
 279 package_go_binary cmd/arvados-client arvados-client \
 280     "Arvados command line tool (beta)"
 281 package_go_binary cmd/arvados-server arvados-server \
 282     "Arvados server daemons"
 283 package_go_binary cmd/arvados-server arvados-controller \
 284     "Arvados cluster controller daemon"
 285 package_go_binary cmd/arvados-server arvados-dispatch-cloud \
 286     "Arvados cluster cloud dispatch"
 287 package_go_binary cmd/arvados-server arvados-dispatch-lsf \
 288     "Dispatch Arvados containers to an LSF cluster"
 289 package_go_binary services/arv-git-httpd arvados-git-httpd \
 290     "Provide authenticated http access to Arvados-hosted git repositories"
 291 package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
 292     "Dispatch Crunch containers on the local system"
 293 package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
 294     "Dispatch Crunch containers to a SLURM cluster"
 295 package_go_binary cmd/arvados-server crunch-run \
 296     "Supervise a single Crunch container"
 297 package_go_binary services/crunchstat crunchstat \
 298     "Gather cpu/memory/network statistics of running Crunch jobs"
 299 package_go_binary services/health arvados-health \
 300     "Check health of all Arvados cluster services"
 301 package_go_binary services/keep-balance keep-balance \
 302     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
 303 package_go_binary services/keepproxy keepproxy \
 304     "Make a Keep cluster accessible to clients that are not on the LAN"
 305 package_go_binary cmd/arvados-server keepstore \
 306     "Keep storage daemon, accessible to clients on the LAN"
 307 package_go_binary services/keep-web keep-web \
 308     "Static web hosting service for user data stored in Arvados Keep"
 309 package_go_binary cmd/arvados-server arvados-ws \
 310     "Arvados Websocket server"
 311 package_go_binary tools/sync-groups arvados-sync-groups \
 312     "Synchronize remote groups into Arvados from an external source"
 313 package_go_binary tools/keep-block-check keep-block-check \
 314     "Verify that all data from one set of Keep servers to another was copied"
 315 package_go_binary tools/keep-rsync keep-rsync \
 316     "Copy all data from one set of Keep servers to another"
 317 package_go_binary tools/keep-exercise keep-exercise \
 318     "Performance testing tool for Arvados Keep"
 319 package_go_so lib/pam pam_arvados.so libpam-arvados-go \
 320     "Arvados PAM authentication module"
 321
 322 # The Python SDK - Python3 package
 323 fpm_build_virtualenv "arvados-python-client" "sdk/python" "python3"
 324
 325 # Arvados cwl runner - Python3 package
 326 fpm_build_virtualenv "arvados-cwl-runner" "sdk/cwl" "python3"
 327
 328 # The FUSE driver - Python3 package
 329 fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
 330
 331 # The Arvados crunchstat-summary tool
 332 fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
 333
 334 # The Docker image cleaner
 335 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
 336
 337 # The Arvados user activity tool
 338 fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
 339
 340 # The python->python3 metapackages
 341 build_metapackage "arvados-fuse" "services/fuse"
 342 build_metapackage "arvados-python-client" "services/fuse"
 343 build_metapackage "arvados-cwl-runner" "sdk/cwl"
 344 build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
 345 build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
 346 build_metapackage "arvados-user-activity" "tools/user-activity"
 347
 348 if [[ -z "$ONLY_BUILD" ]] || [[ "cwltest" == "$ONLY_BUILD" ]] ; then
 349   # The cwltest package, which lives out of tree
 350   cd "$WORKSPACE"
 351   if [[ -e "$WORKSPACE/cwltest" ]]; then
 352     rm -rf "$WORKSPACE/cwltest"
 353   fi
 354   git clone https://github.com/common-workflow-language/cwltest.git
 355   # signal to our build script that we want a cwltest executable installed in /usr/bin/
 356   mkdir cwltest/bin && touch cwltest/bin/cwltest
 357   fpm_build_virtualenv "cwltest" "cwltest" "python3"
 358   # The python->python3 metapackage
 359   build_metapackage "cwltest" "cwltest"
 360   cd "$WORKSPACE"
 361   rm -rf "$WORKSPACE/cwltest"
 362 fi
 363
 364 calculate_go_package_version arvados_server_version cmd/arvados-server
 365 arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
 366
 367 # Build the API server package
 368 test_rails_package_presence arvados-api-server "$WORKSPACE/services/api"
 369 if [[ "$?" == "0" ]]; then
 370   handle_rails_package arvados-api-server "$WORKSPACE/services/api" \
 371       "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
 372       --description="Arvados API server - Arvados is a free and open source platform for big data science." \
 373       --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
 374 fi
 375
 376 # Build the workbench server package
 377 if [[ "$HOSTTYPE" == "x86_64" ]]; then
 378   test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
 379   if [[ "$?" == "0" ]] ; then
 380     (
 381         set -e
 382
 383         # The workbench package has a build-time dependency on the arvados-server
 384         # package for config manipulation, so install it first.
 385         cd $WORKSPACE/cmd/arvados-server
 386         get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
 387
 388         arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
 389         if [[ ! -e ${arvados_server_pkg_path} ]]; then
 390           arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
 391         fi
 392         if [[ "$FORMAT" == "deb" ]]; then
 393           dpkg -i ${arvados_server_pkg_path}
 394         else
 395           rpm -i ${arvados_server_pkg_path}
 396         fi
 397
 398         cd "$WORKSPACE/apps/workbench"
 399
 400         # We need to bundle to be ready even when we build a package without vendor directory
 401         # because asset compilation requires it.
 402         bundle install --system >"$STDOUT_IF_DEBUG"
 403
 404         # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
 405         # and we want that in the package, so it's easier to not exclude the tmp directory
 406         # from the package - empty it instead.
 407         rm -rf tmp
 408         mkdir tmp
 409
 410         # Set up an appropriate config.yml
 411         arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
 412         mkdir -p /etc/arvados/
 413         mv /tmp/x /etc/arvados/config.yml
 414         perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
 415
 416         ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
 417         ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
 418
 419         # Remove generated configuration files so they don't go in the package.
 420         rm -rf /etc/arvados/
 421     )
 422
 423     if [[ "$?" != "0" ]]; then
 424       echo "ERROR: Asset precompilation failed"
 425       EXITCODE=1
 426     else
 427       handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
 428           "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
 429           --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
 430           --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
 431     fi
 432   fi
 433 else
 434   echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
 435 fi
 436
 437 # clean up temporary GOPATH
 438 rm -rf "$GOPATH"
 439
 440 exit $EXITCODE