X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b1a2bad108376bf0cf50e4260e47c7047318ab6c..964c763379e2ea98e46584267342ced694e3349a:/build/run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 1076095900..92140bbdc8 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -130,10 +130,12 @@ calculate_go_package_version() {
   __returnvar="$version"
 }
 
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt]
+# Usage: package_go_binary services/foo arvados-foo [deb|rpm] [amd64|arm64] "Compute foo to arbitrary precision" [apache-2.0.txt]
 package_go_binary() {
     local src_path="$1"; shift
     local prog="$1"; shift
+    local package_format="$1"; shift
+    local target_arch="$1"; shift
     local description="$1"; shift
     local license_file="${1:-agpl-3.0.txt}"; shift
 
@@ -145,21 +147,78 @@ package_go_binary() {
       fi
     fi
 
-    debug_echo "package_go_binary $src_path as $prog"
+    # Only amd64 and aarch64 are supported
+    native_arch="amd64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+        native_arch="arm64"
+    fi
+
+    if [[ "$native_arch" != "amd64" ]] && [[ -n "$target_arch" ]] && [[ "$native_arch" != "$target_arch" ]]; then
+      echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $target_arch"
+      return 1
+    fi
+
+    if [[ -n "$target_arch" ]]; then
+      # A target architecture has been specified
+      package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$target_arch" "$license_file"
+      return $?
+    else
+      # No target architecture specified, default to native target. When on amd64 also crosscompile arm64
+      # but only when building deb packages (centos does not have support for crosscompiling userspace).
+      archs=($native_arch)
+      if [[ "$native_arch" == "amd64" ]] && [[ "$package_format" == "deb" ]]; then
+        archs=('amd64' 'arm64')
+      fi
+      for ta in $archs; do
+        package_go_binary_worker "$src_path" "$prog" "$package_format" "$description" "$native_arch" "$ta" "$license_file"
+        retval=$?
+        if [[ "$retval" != 0 ]]; then
+          return $retval
+        fi
+      done
+    fi
+}
 
+# Usage: package_go_binary services/foo arvados-foo deb "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
+package_go_binary_worker() {
+    local src_path="$1"; shift
+    local prog="$1"; shift
+    local package_format="$1"; shift
+    local description="$1"; shift
+    local native_arch="${1:-amd64}"; shift
+    local target_arch="${1:-amd64}"; shift
+    local license_file="${1:-agpl-3.0.txt}"; shift
+
+    if [[ "$native_arch" != "$target_arch" ]] && [[ "$package_format" == "rpm" ]]; then
+      echo "Error: no cross compilation support for Go on $native_arch ($package_format), can not build $prog for $target_arch"
+      return 1
+    fi
+
+    debug_echo "package_go_binary $src_path as $prog (native arch: $native_arch, target arch: $target_arch)"
     local basename="${src_path##*/}"
     calculate_go_package_version go_package_version $src_path
 
     cd $WORKSPACE/packages/$TARGET
-    test_package_presence $prog $go_package_version go
-
+    test_package_presence "$prog" "$go_package_version" "go" "" "$target_arch"
     if [[ "$?" != "0" ]]; then
       return 1
     fi
 
-    go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+    echo "Building $pachage_format ($target_arch) package for $prog from $src_path"
+    if [[ "$native_arch" == "amd64" ]] && [[ "$target_arch" == "arm64" ]]; then
+      CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${target_arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+    else
+      GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+    fi
 
     local -a switches=()
+
+    binpath=$GOPATH/bin/${basename}
+    if [[ "${target_arch}" != "${native_arch}" ]]; then
+      switches+=("-a${target_arch}")
+      binpath="$GOPATH/bin/linux_${target_arch}/${basename}"
+    fi
+
     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
     if [[ -e "${systemd_unit}" ]]; then
         switches+=(
@@ -169,16 +228,22 @@ package_go_binary() {
     fi
     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
 
-    fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+    fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
 }
 
-# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
+# Usage: package_go_so lib/foo arvados_foo.so arvados-foo deb amd64 "Arvados foo library"
 package_go_so() {
     local src_path="$1"; shift
     local sofile="$1"; shift
     local pkg="$1"; shift
+    local package_format="$1"; shift
+    local target_arch="$1"; shift # supported: amd64, arm64
     local description="$1"; shift
 
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+      return 0
+    fi
+
     debug_echo "package_go_so $src_path as $pkg"
 
     calculate_go_package_version go_package_version $src_path
@@ -282,8 +347,14 @@ get_complete_package_name() {
   fi
 
   if [[ "$arch" == "" ]]; then
-    rpm_architecture="x86_64"
-    deb_architecture="amd64"
+    native_arch="amd64"
+    rpm_native_arch="x86_64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+      native_arch="arm64"
+      rpm_native_arch="arm64"
+    fi
+    rpm_architecture="$rpm_native_arch"
+    deb_architecture="$native_arch"
 
     if [[ "$pkgtype" =~ ^(src)$ ]]; then
       rpm_architecture="noarch"
@@ -292,8 +363,8 @@ get_complete_package_name() {
 
     # These python packages have binary components
     if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
-      rpm_architecture="x86_64"
-      deb_architecture="amd64"
+      rpm_architecture="$rpm_native_arch"
+      deb_architecture="$native_arch"
     fi
   else
     rpm_architecture=$arch
@@ -325,7 +396,7 @@ test_package_presence() {
     fi
 
     local full_pkgname
-    get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch
+    get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
 
     # See if we can skip building the package, only if it already exists in the
     # processed/ directory. If so, move it back to the packages directory to make
@@ -337,7 +408,7 @@ test_package_presence() {
     elif [[ "$FORMAT" == "deb" ]]; then
       declare -A dd
       dd[debian10]=buster
-      dd[ubuntu1604]=xenial
+      dd[debian11]=bullseye
       dd[ubuntu1804]=bionic
       dd[ubuntu2004]=focal
       D=${dd[$TARGET]}
@@ -439,6 +510,32 @@ fpm_build_virtualenv () {
   shift
   PACKAGE_TYPE=${1:-python}
   shift
+  native_arch="amd64"
+  if [[ "$HOSTTYPE" == "aarch64" ]]; then
+    native_arch="arm64"
+  fi
+
+  if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
+  elif [[ -z "$ARCH" ]]; then
+    for arch in $native_arch; do
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
+    done
+  else
+    echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+  fi
+}
+
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv_worker () {
+  PKG=$1
+  shift
+  PKG_DIR=$1
+  shift
+  PACKAGE_TYPE=${1:-python}
+  shift
+  arch=${1:-amd64}
+  shift
 
   # Set up
   STDOUT_IF_DEBUG=/dev/null
@@ -514,11 +611,11 @@ fpm_build_virtualenv () {
   # We can't do this earlier than here, because we need PYTHON_VERSION...
   # This isn't so bad; the sdist call above is pretty quick compared to
   # the invocation of virtualenv and fpm, below.
-  if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+  if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
     return 0
   fi
 
-  echo "Building $FORMAT package for $PKG from $PKG_DIR"
+  echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
 
   # Package the sdist in a virtualenv
   echo "Creating virtualenv..."
@@ -603,6 +700,10 @@ fpm_build_virtualenv () {
 
   declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
 
+  if [[ "${arch}" != "amd64" ]]; then
+    COMMAND_ARR+=("-a${arch}")
+  fi
+
   if [[ "$MAINTAINER" != "" ]]; then
     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
   fi
@@ -621,6 +722,11 @@ fpm_build_virtualenv () {
   LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
   COMMAND_ARR+=('--license' "$LICENSE_STRING")
 
+  if [[ "$FORMAT" == "rpm" ]]; then
+    # Make sure to conflict with the old rh-python36 packages we used to publish
+    COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
+  fi
+
   if [[ "$DEBUG" != "0" ]]; then
     COMMAND_ARR+=('--verbose' '--log' 'info')
   fi
@@ -685,8 +791,8 @@ fpm_build_virtualenv () {
   fi
 
   # the python3-arvados-cwl-runner package comes with cwltool, expose that version
-  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
-    COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
+    COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
   fi
 
   COMMAND_ARR+=(".")
@@ -720,15 +826,19 @@ build_metapackage() {
     return 0
   fi
 
+  if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+    ARVADOS_BUILDING_ITERATION=1
+  fi
+
   if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
     cd $WORKSPACE/$PKG_DIR
     pwd
     rm -rf dist/*
 
     # Get the latest setuptools
-    if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
-      echo "Error, unable to upgrade setuptools with"
-      echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+    if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+      echo "Error, unable to upgrade setuptools with XY"
+      echo "  pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
       exit 1
     fi
     # filter a useless warning (when building the cwltest package) from the stderr output
@@ -903,6 +1013,7 @@ fpm_build () {
   declare -a fpm_args=()
   declare -a build_depends=()
   declare -a fpm_depends=()
+  declare -a fpm_conflicts=()
   declare -a fpm_exclude=()
   if [[ ! -d "$SRC_DIR" ]]; then
       echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
@@ -938,6 +1049,9 @@ fpm_build () {
   for i in "${fpm_depends[@]}"; do
     COMMAND_ARR+=('--depends' "$i")
   done
+  for i in "${fpm_conflicts[@]}"; do
+    COMMAND_ARR+=('--conflicts' "$i")
+  done
   for i in "${fpm_exclude[@]}"; do
     COMMAND_ARR+=('--exclude' "$i")
   done