X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ad92a003eb3c4efdd3f880b5301033b755d6d247..964c763379e2ea98e46584267342ced694e3349a:/build/run-build-packages.sh

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 8d55e2fd9b..2e22a9e53b 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -22,6 +22,8 @@ Options:
     Distribution to build packages for (default: debian10)
 --only-build <package>
     Build only a specific package (or $ONLY_BUILD from environment)
+--arch <arch>
+    Build a specific architecture (or $ARCH from environment, defaults to native architecture)
 --force-build
     Build even if the package exists upstream or if it has already been
     built locally
@@ -50,7 +52,7 @@ TARGET=debian10
 COMMAND=
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,build-bundle-packages,debug,target:,only-build:,force-build \
+    help,build-bundle-packages,debug,target:,only-build:,arch:,force-build \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -73,6 +75,9 @@ while [ $# -gt 0 ]; do
         --force-build)
             FORCE_BUILD=1
             ;;
+        --arch)
+            ARCH="$2"; shift
+            ;;
         --debug)
             DEBUG=1
             ;;
@@ -238,78 +243,80 @@ handle_ruby_gem arvados-login-sync
 # Python packages
 debug_echo -e "\nPython packages\n"
 
-# arvados-src
-(
-    cd "$WORKSPACE"
-    COMMIT_HASH=$(format_last_commit_here "%H")
-    arvados_src_version="$(version_from_git)"
-
-    cd $WORKSPACE/packages/$TARGET
-    test_package_presence arvados-src $arvados_src_version src ""
-
-    if [[ "$?" == "0" ]]; then
+if [[ -z "$ONLY_BUILD" ]] || [[ "arvados-src" == "$ONLY_BUILD" ]] ; then
+  # arvados-src
+  (
       cd "$WORKSPACE"
-      SRC_BUILD_DIR=$(mktemp -d)
-      # mktemp creates the directory with 0700 permissions by default
-      chmod 755 $SRC_BUILD_DIR
-      git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
-      cd "$SRC_BUILD_DIR"
-
-      # go into detached-head state
-      git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
-      echo "$COMMIT_HASH" >git-commit.version
-
-      cd "$SRC_BUILD_DIR"
-      PKG_VERSION=$(version_from_git)
+      COMMIT_HASH=$(format_last_commit_here "%H")
+      arvados_src_version="$(version_from_git)"
+
       cd $WORKSPACE/packages/$TARGET
-      fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+      test_package_presence arvados-src "$arvados_src_version" src ""
 
-      rm -rf "$SRC_BUILD_DIR"
-    fi
-)
+      if [[ "$?" == "0" ]]; then
+        cd "$WORKSPACE"
+        SRC_BUILD_DIR=$(mktemp -d)
+        # mktemp creates the directory with 0700 permissions by default
+        chmod 755 $SRC_BUILD_DIR
+        git clone $DASHQ_UNLESS_DEBUG "$WORKSPACE/.git" "$SRC_BUILD_DIR"
+        cd "$SRC_BUILD_DIR"
+
+        # go into detached-head state
+        git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
+        echo "$COMMIT_HASH" >git-commit.version
+
+        cd $WORKSPACE/packages/$TARGET
+        fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+
+        rm -rf "$SRC_BUILD_DIR"
+      fi
+  )
+fi
 
 # Go binaries
 cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
-package_go_binary cmd/arvados-client arvados-client \
+package_go_binary cmd/arvados-client arvados-client "$FORMAT" "$ARCH" \
     "Arvados command line tool (beta)"
-package_go_binary cmd/arvados-server arvados-server \
+package_go_binary cmd/arvados-server arvados-server "$FORMAT" "$ARCH" \
     "Arvados server daemons"
-package_go_binary cmd/arvados-server arvados-controller \
+package_go_binary cmd/arvados-server arvados-controller "$FORMAT" "$ARCH" \
     "Arvados cluster controller daemon"
-package_go_binary cmd/arvados-server arvados-dispatch-cloud \
+package_go_binary cmd/arvados-server arvados-dispatch-cloud "$FORMAT" "$ARCH" \
     "Arvados cluster cloud dispatch"
-package_go_binary services/arv-git-httpd arvados-git-httpd \
+package_go_binary cmd/arvados-server arvados-dispatch-lsf "$FORMAT" "$ARCH" \
+    "Dispatch Arvados containers to an LSF cluster"
+package_go_binary services/arv-git-httpd arvados-git-httpd "$FORMAT" "$ARCH" \
     "Provide authenticated http access to Arvados-hosted git repositories"
-package_go_binary services/crunch-dispatch-local crunch-dispatch-local \
+package_go_binary services/crunch-dispatch-local crunch-dispatch-local "$FORMAT" "$ARCH" \
     "Dispatch Crunch containers on the local system"
-package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm \
+package_go_binary services/crunch-dispatch-slurm crunch-dispatch-slurm "$FORMAT" "$ARCH" \
     "Dispatch Crunch containers to a SLURM cluster"
-package_go_binary cmd/arvados-server crunch-run \
+package_go_binary cmd/arvados-server crunch-run "$FORMAT" "$ARCH" \
     "Supervise a single Crunch container"
-package_go_binary services/crunchstat crunchstat \
+package_go_binary services/crunchstat crunchstat "$FORMAT" "$ARCH" \
     "Gather cpu/memory/network statistics of running Crunch jobs"
-package_go_binary services/health arvados-health \
+package_go_binary services/health arvados-health "$FORMAT" "$ARCH" \
     "Check health of all Arvados cluster services"
-package_go_binary services/keep-balance keep-balance \
+package_go_binary services/keep-balance keep-balance "$FORMAT" "$ARCH" \
     "Rebalance and garbage-collect data blocks stored in Arvados Keep"
-package_go_binary services/keepproxy keepproxy \
+package_go_binary services/keepproxy keepproxy "$FORMAT" "$ARCH" \
     "Make a Keep cluster accessible to clients that are not on the LAN"
-package_go_binary services/keepstore keepstore \
+package_go_binary cmd/arvados-server keepstore "$FORMAT" "$ARCH" \
     "Keep storage daemon, accessible to clients on the LAN"
-package_go_binary services/keep-web keep-web \
+package_go_binary services/keep-web keep-web "$FORMAT" "$ARCH" \
     "Static web hosting service for user data stored in Arvados Keep"
-package_go_binary cmd/arvados-server arvados-ws \
+package_go_binary cmd/arvados-server arvados-ws "$FORMAT" "$ARCH" \
     "Arvados Websocket server"
-package_go_binary tools/sync-groups arvados-sync-groups \
+package_go_binary tools/sync-groups arvados-sync-groups "$FORMAT" "$ARCH" \
     "Synchronize remote groups into Arvados from an external source"
-package_go_binary tools/keep-block-check keep-block-check \
+package_go_binary tools/keep-block-check keep-block-check "$FORMAT" "$ARCH" \
     "Verify that all data from one set of Keep servers to another was copied"
-package_go_binary tools/keep-rsync keep-rsync \
+package_go_binary tools/keep-rsync keep-rsync "$FORMAT" "$ARCH" \
     "Copy all data from one set of Keep servers to another"
-package_go_binary tools/keep-exercise keep-exercise \
+package_go_binary tools/keep-exercise keep-exercise "$FORMAT" "$ARCH" \
     "Performance testing tool for Arvados Keep"
-package_go_so lib/pam pam_arvados.so libpam-arvados-go \
+package_go_so lib/pam pam_arvados.so libpam-arvados-go "$FORMAT" "$ARCH" \
     "Arvados PAM authentication module"
 
 # The Python SDK - Python3 package
@@ -327,16 +334,32 @@ fpm_build_virtualenv "crunchstat-summary" "tools/crunchstat-summary" "python3"
 # The Docker image cleaner
 fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
 
-# The cwltest package, which lives out of tree
-cd "$WORKSPACE"
-if [[ -e "$WORKSPACE/cwltest" ]]; then
-	rm -rf "$WORKSPACE/cwltest"
+# The Arvados user activity tool
+fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
+
+# The python->python3 metapackages
+build_metapackage "arvados-fuse" "services/fuse"
+build_metapackage "arvados-python-client" "services/fuse"
+build_metapackage "arvados-cwl-runner" "sdk/cwl"
+build_metapackage "crunchstat-summary" "tools/crunchstat-summary"
+build_metapackage "arvados-docker-cleaner" "services/dockercleaner"
+build_metapackage "arvados-user-activity" "tools/user-activity"
+
+if [[ -z "$ONLY_BUILD" ]] || [[ "cwltest" == "$ONLY_BUILD" ]] ; then
+  # The cwltest package, which lives out of tree
+  cd "$WORKSPACE"
+  if [[ -e "$WORKSPACE/cwltest" ]]; then
+    rm -rf "$WORKSPACE/cwltest"
+  fi
+  git clone https://github.com/common-workflow-language/cwltest.git
+  # signal to our build script that we want a cwltest executable installed in /usr/bin/
+  mkdir cwltest/bin && touch cwltest/bin/cwltest
+  fpm_build_virtualenv "cwltest" "cwltest" "python3"
+  # The python->python3 metapackage
+  build_metapackage "cwltest" "cwltest"
+  cd "$WORKSPACE"
+  rm -rf "$WORKSPACE/cwltest"
 fi
-git clone https://github.com/common-workflow-language/cwltest.git
-# signal to our build script that we want a cwltest executable installed in /usr/bin/
-mkdir cwltest/bin && touch cwltest/bin/cwltest
-fpm_build_virtualenv "cwltest" "cwltest" "python3"
-rm -rf "$WORKSPACE/cwltest"
 
 calculate_go_package_version arvados_server_version cmd/arvados-server
 arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go")
@@ -351,60 +374,64 @@ if [[ "$?" == "0" ]]; then
 fi
 
 # Build the workbench server package
-test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
-if [[ "$?" == "0" ]] ; then
-  (
-      set -e
-
-      # The workbench package has a build-time dependency on the arvados-server
-      # package for config manipulation, so install it first.
-      cd $WORKSPACE/cmd/arvados-server
-      get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
-      arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
-      if [[ ! -e ${arvados_server_pkg_path} ]]; then
-        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
-      fi
-      if [[ "$FORMAT" == "deb" ]]; then
-        dpkg -i ${arvados_server_pkg_path}
-      else
-        rpm -i ${arvados_server_pkg_path}
-      fi
-
-      cd "$WORKSPACE/apps/workbench"
-
-      # We need to bundle to be ready even when we build a package without vendor directory
-      # because asset compilation requires it.
-      bundle install --system >"$STDOUT_IF_DEBUG"
-
-      # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
-      # and we want that in the package, so it's easier to not exclude the tmp directory
-      # from the package - empty it instead.
-      rm -rf tmp
-      mkdir tmp
-
-      # Set up an appropriate config.yml
-      arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
-      mkdir -p /etc/arvados/
-      mv /tmp/x /etc/arvados/config.yml
-      perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
-      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >"$STDOUT_IF_DEBUG"
-      RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >"$STDOUT_IF_DEBUG"
-
-      # Remove generated configuration files so they don't go in the package.
-      rm -rf /etc/arvados/
-  )
-
-  if [[ "$?" != "0" ]]; then
-    echo "ERROR: Asset precompilation failed"
-    EXITCODE=1
-  else
-    handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
-        "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
-        --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
-        --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+if [[ "$HOSTTYPE" == "x86_64" ]]; then
+  test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+  if [[ "$?" == "0" ]] ; then
+    (
+        set -e
+
+        # The workbench package has a build-time dependency on the arvados-server
+        # package for config manipulation, so install it first.
+        cd $WORKSPACE/cmd/arvados-server
+        get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+        arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+        if [[ ! -e ${arvados_server_pkg_path} ]]; then
+          arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+        fi
+        if [[ "$FORMAT" == "deb" ]]; then
+          dpkg -i ${arvados_server_pkg_path}
+        else
+          rpm -i ${arvados_server_pkg_path}
+        fi
+
+        cd "$WORKSPACE/apps/workbench"
+
+        # We need to bundle to be ready even when we build a package without vendor directory
+        # because asset compilation requires it.
+        bundle install --system >"$STDOUT_IF_DEBUG"
+
+        # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+        # and we want that in the package, so it's easier to not exclude the tmp directory
+        # from the package - empty it instead.
+        rm -rf tmp
+        mkdir tmp
+
+        # Set up an appropriate config.yml
+        arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo  "Clusters: {zzzzz: {}}") > /tmp/x
+        mkdir -p /etc/arvados/
+        mv /tmp/x /etc/arvados/config.yml
+        perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+        ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+        # Remove generated configuration files so they don't go in the package.
+        rm -rf /etc/arvados/
+    )
+
+    if [[ "$?" != "0" ]]; then
+      echo "ERROR: Asset precompilation failed"
+      EXITCODE=1
+    else
+      handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+          "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+          --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+          --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+    fi
   fi
+else
+  echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
 fi
 
 # clean up temporary GOPATH