X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9516ddc7158e077743a0daffb36e77a16a59c736..4db7a388ce1543d013ac5bfe97f7431eae95d422:/build/run-library.sh

diff --git a/build/run-library.sh b/build/run-library.sh
index 59a9e4f47d..eb7f086278 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -145,21 +145,67 @@ package_go_binary() {
       fi
     fi
 
-    debug_echo "package_go_binary $src_path as $prog"
+    native_arch="amd64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+        native_arch="arm64"
+    fi
+
+    if [[ -n "$ARCH" ]]; then
+      if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ARCH" ]]; then
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ARCH" "$license_file"
+      else
+        echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ARCH"
+      fi
+    else
+      archs=($native_arch)
+      if [[ "$native_arch" == "amd64" ]]; then
+        archs=('amd64' 'arm64')
+      fi
+      for arch in $archs; do
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file"
+      done
+    fi
+}
+
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
+package_go_binary_worker() {
+    local src_path="$1"; shift
+    local prog="$1"; shift
+    local description="$1"; shift
+    local native_arch="${1:-amd64}"; shift
+    local arch="${1:-amd64}"; shift
+    local license_file="${1:-agpl-3.0.txt}"; shift
 
+    debug_echo "package_go_binary $src_path as $prog"
     local basename="${src_path##*/}"
     calculate_go_package_version go_package_version $src_path
 
     cd $WORKSPACE/packages/$TARGET
-    test_package_presence $prog $go_package_version go
-
+    test_package_presence "$prog" "$go_package_version" "go" "" "$arch"
     if [[ "$?" != "0" ]]; then
       return 1
     fi
 
-    go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+    echo "BUILDING ${arch}"
+    if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then
+      if [[ "$FORMAT" == "deb" ]]; then
+        CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+      else
+        echo "Error: no cross compilation support for Go on $native_arch ($FORMAT), can not build $prog for $ARCH"
+        return
+      fi
+    else
+      GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
+    fi
 
     local -a switches=()
+
+    binpath=$GOPATH/bin/${basename}
+    if [[ "${arch}" != "${native_arch}" ]]; then
+      switches+=("-a${arch}")
+      binpath="$GOPATH/bin/linux_${arch}/${basename}"
+    fi
+
     systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
     if [[ -e "${systemd_unit}" ]]; then
         switches+=(
@@ -169,7 +215,7 @@ package_go_binary() {
     fi
     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
 
-    fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+    fpm_build "${WORKSPACE}/${src_path}" "$binpath=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
 }
 
 # Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
@@ -179,6 +225,10 @@ package_go_so() {
     local pkg="$1"; shift
     local description="$1"; shift
 
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$pkg" != "$ONLY_BUILD" ]]; then
+      return 0
+    fi
+
     debug_echo "package_go_so $src_path as $pkg"
 
     calculate_go_package_version go_package_version $src_path
@@ -282,8 +332,14 @@ get_complete_package_name() {
   fi
 
   if [[ "$arch" == "" ]]; then
-    rpm_architecture="x86_64"
-    deb_architecture="amd64"
+    native_arch="amd64"
+    rpm_native_arch="x86_64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+      native_arch="arm64"
+      rpm_native_arch="arm64"
+    fi
+    rpm_architecture="$rpm_native_arch"
+    deb_architecture="$native_arch"
 
     if [[ "$pkgtype" =~ ^(src)$ ]]; then
       rpm_architecture="noarch"
@@ -292,8 +348,8 @@ get_complete_package_name() {
 
     # These python packages have binary components
     if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
-      rpm_architecture="x86_64"
-      deb_architecture="amd64"
+      rpm_architecture="$rpm_native_arch"
+      deb_architecture="$native_arch"
     fi
   else
     rpm_architecture=$arch
@@ -325,7 +381,7 @@ test_package_presence() {
     fi
 
     local full_pkgname
-    get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch
+    get_complete_package_name full_pkgname "$pkgname" "$version" "$pkgtype" "$iteration" "$arch"
 
     # See if we can skip building the package, only if it already exists in the
     # processed/ directory. If so, move it back to the packages directory to make
@@ -337,7 +393,7 @@ test_package_presence() {
     elif [[ "$FORMAT" == "deb" ]]; then
       declare -A dd
       dd[debian10]=buster
-      dd[ubuntu1604]=xenial
+      dd[debian11]=bullseye
       dd[ubuntu1804]=bionic
       dd[ubuntu2004]=focal
       D=${dd[$TARGET]}
@@ -439,6 +495,32 @@ fpm_build_virtualenv () {
   shift
   PACKAGE_TYPE=${1:-python}
   shift
+  native_arch="amd64"
+  if [[ "$HOSTTYPE" == "aarch64" ]]; then
+    native_arch="arm64"
+  fi
+
+  if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
+  elif [[ -z "$ARCH" ]]; then
+    for arch in $native_arch; do
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
+    done
+  else
+    echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+  fi
+}
+
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv_worker () {
+  PKG=$1
+  shift
+  PKG_DIR=$1
+  shift
+  PACKAGE_TYPE=${1:-python}
+  shift
+  arch=${1:-amd64}
+  shift
 
   # Set up
   STDOUT_IF_DEBUG=/dev/null
@@ -514,11 +596,11 @@ fpm_build_virtualenv () {
   # We can't do this earlier than here, because we need PYTHON_VERSION...
   # This isn't so bad; the sdist call above is pretty quick compared to
   # the invocation of virtualenv and fpm, below.
-  if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+  if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
     return 0
   fi
 
-  echo "Building $FORMAT package for $PKG from $PKG_DIR"
+  echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
 
   # Package the sdist in a virtualenv
   echo "Creating virtualenv..."
@@ -603,6 +685,10 @@ fpm_build_virtualenv () {
 
   declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
 
+  if [[ "${arch}" != "amd64" ]]; then
+    COMMAND_ARR+=("-a${arch}")
+  fi
+
   if [[ "$MAINTAINER" != "" ]]; then
     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
   fi
@@ -621,9 +707,10 @@ fpm_build_virtualenv () {
   LICENSE_STRING=`grep license $WORKSPACE/$PKG_DIR/setup.py|cut -f2 -d=|sed -e "s/[',\\"]//g"`
   COMMAND_ARR+=('--license' "$LICENSE_STRING")
 
-  #if [[ "$FORMAT" != "rpm" ]]; then
-  #  COMMAND_ARR+=('--conflicts' "python-$PKG")
-  #fi
+  if [[ "$FORMAT" == "rpm" ]]; then
+    # Make sure to conflict with the old rh-python36 packages we used to publish
+    COMMAND_ARR+=('--conflicts' "rh-python36-python-$PKG")
+  fi
 
   if [[ "$DEBUG" != "0" ]]; then
     COMMAND_ARR+=('--verbose' '--log' 'info')
@@ -664,6 +751,10 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--depends' "$i")
   done
 
+  for i in "${fpm_depends[@]}"; do
+    COMMAND_ARR+=('--replaces' "python-$PKG")
+  done
+
   # make sure the systemd service file ends up in the right place
   # used by arvados-docker-cleaner
   if [[ -e "${systemd_unit}" ]]; then
@@ -685,8 +776,8 @@ fpm_build_virtualenv () {
   fi
 
   # the python3-arvados-cwl-runner package comes with cwltool, expose that version
-  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
-    COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+  if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/$PYTHON_PKG/bin/cwltool" ]]; then
+    COMMAND_ARR+=("usr/share/$python/dist/$PYTHON_PKG/bin/cwltool=/usr/bin/")
   fi
 
   COMMAND_ARR+=(".")
@@ -720,15 +811,19 @@ build_metapackage() {
     return 0
   fi
 
+  if [[ "$ARVADOS_BUILDING_ITERATION" == "" ]]; then
+    ARVADOS_BUILDING_ITERATION=1
+  fi
+
   if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
     cd $WORKSPACE/$PKG_DIR
     pwd
     rm -rf dist/*
 
     # Get the latest setuptools
-    if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
-      echo "Error, unable to upgrade setuptools with"
-      echo "  $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
+    if ! pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then
+      echo "Error, unable to upgrade setuptools with XY"
+      echo "  pip3 install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'"
       exit 1
     fi
     # filter a useless warning (when building the cwltest package) from the stderr output
@@ -903,6 +998,7 @@ fpm_build () {
   declare -a fpm_args=()
   declare -a build_depends=()
   declare -a fpm_depends=()
+  declare -a fpm_conflicts=()
   declare -a fpm_exclude=()
   if [[ ! -d "$SRC_DIR" ]]; then
       echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
@@ -938,6 +1034,9 @@ fpm_build () {
   for i in "${fpm_depends[@]}"; do
     COMMAND_ARR+=('--depends' "$i")
   done
+  for i in "${fpm_conflicts[@]}"; do
+    COMMAND_ARR+=('--conflicts' "$i")
+  done
   for i in "${fpm_exclude[@]}"; do
     COMMAND_ARR+=('--exclude' "$i")
   done