17417: add native arm64 build support to our package build images. Add
[arvados.git] / build / run-library.sh
index dd8e4b274702e4a1e5a38c7dc18fd231092409b6..fd04c25ed937508b2ccb1e827af808eb8aa62edf 100755 (executable)
@@ -137,31 +137,45 @@ package_go_binary() {
     local description="$1"; shift
     local license_file="${1:-agpl-3.0.txt}"; shift
 
-    if [[ -n "$ONLY_ARCH" ]]; then
-        package_go_binary_worker "$src_path" "$prog" "$description" "$ONLY_ARCH" "$license_file"
+    if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+      # arvados-workbench depends on arvados-server at build time, so even when
+      # only arvados-workbench is being built, we need to build arvados-server too
+      if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+        return 0
+      fi
+    fi
+
+    native_arch="amd64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+        native_arch="arm64"
+    fi
+
+    if [[ -n "$ARCH" ]]; then
+      if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ARCH" ]]; then
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ARCH" "$license_file"
+      else
+        echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ARCH"
+      fi
     else
-      for arch in 'amd64' 'arm64'; do
-        package_go_binary_worker "$src_path" "$prog" "$description" "$arch" "$license_file"
+      archs=($native_arch)
+      if [[ "$native_arch" == "amd64" ]]; then
+        archs=('amd64' 'arm64')
+      fi
+      for arch in $archs; do
+        package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file"
       done
     fi
 }
 
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [apache-2.0.txt]
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [amd64/arm64] [amd64/arm64] [apache-2.0.txt]
 package_go_binary_worker() {
     local src_path="$1"; shift
     local prog="$1"; shift
     local description="$1"; shift
+    local native_arch="${1:-amd64}"; shift
     local arch="${1:-amd64}"; shift
     local license_file="${1:-agpl-3.0.txt}"; shift
 
-    if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
-      # arvados-workbench depends on arvados-server at build time, so even when
-      # only arvados-workbench is being built, we need to build arvados-server too
-      if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
-        return 0
-      fi
-    fi
-
     debug_echo "package_go_binary $src_path as $prog"
     local basename="${src_path##*/}"
     calculate_go_package_version go_package_version $src_path
@@ -173,7 +187,7 @@ package_go_binary_worker() {
     fi
 
     echo "BUILDING ${arch}"
-    if [[ "$arch" == "arm64" ]]; then
+    if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then
       CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
     else
       GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
@@ -182,7 +196,7 @@ package_go_binary_worker() {
     local -a switches=()
 
     binpath=$GOPATH/bin/${basename}
-    if [[ "${arch}" != "amd64" ]]; then
+    if [[ "${arch}" != "${native_arch}" ]]; then
       switches+=("-a${arch}")
       binpath="$GOPATH/bin/linux_${arch}/${basename}"
     fi
@@ -313,8 +327,14 @@ get_complete_package_name() {
   fi
 
   if [[ "$arch" == "" ]]; then
-    rpm_architecture="x86_64"
-    deb_architecture="amd64"
+    native_arch="amd64"
+    rpm_native_arch="x86_64"
+    if [[ "$HOSTTYPE" == "aarch64" ]]; then
+      native_arch="arm64"
+      rpm_native_arch="arm64"
+    fi
+    rpm_architecture="$rpm_native_arch"
+    deb_architecture="$native_arch"
 
     if [[ "$pkgtype" =~ ^(src)$ ]]; then
       rpm_architecture="noarch"
@@ -323,8 +343,8 @@ get_complete_package_name() {
 
     # These python packages have binary components
     if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
-      rpm_architecture="x86_64"
-      deb_architecture="amd64"
+      rpm_architecture="$rpm_native_arch"
+      deb_architecture="$native_arch"
     fi
   else
     rpm_architecture=$arch
@@ -470,6 +490,32 @@ fpm_build_virtualenv () {
   shift
   PACKAGE_TYPE=${1:-python}
   shift
+  native_arch="amd64"
+  if [[ "$HOSTTYPE" == "aarch64" ]]; then
+    native_arch="arm64"
+  fi
+
+  if [[ -n "$ARCH" ]] && [[ "$ARCH" == "$native_arch" ]]; then
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$ARCH"
+  elif [[ -z "$ARCH" ]]; then
+    for arch in $native_arch; do
+      fpm_build_virtualenv_worker "$PKG" "$PKG_DIR" "$PACKAGE_TYPE" "$arch"
+    done
+  else
+    echo "Error: no cross compilation support for Python yet, can not build $PKG for $ARCH"
+  fi
+}
+
+# Build python packages with a virtualenv built-in
+fpm_build_virtualenv_worker () {
+  PKG=$1
+  shift
+  PKG_DIR=$1
+  shift
+  PACKAGE_TYPE=${1:-python}
+  shift
+  arch=${1:-amd64}
+  shift
 
   # Set up
   STDOUT_IF_DEBUG=/dev/null
@@ -545,11 +591,11 @@ fpm_build_virtualenv () {
   # We can't do this earlier than here, because we need PYTHON_VERSION...
   # This isn't so bad; the sdist call above is pretty quick compared to
   # the invocation of virtualenv and fpm, below.
-  if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+  if ! test_package_presence "$PYTHON_PKG" "$UNFILTERED_PYTHON_VERSION" "$PACKAGE_TYPE" "$ARVADOS_BUILDING_ITERATION" "$arch"; then
     return 0
   fi
 
-  echo "Building $FORMAT package for $PKG from $PKG_DIR"
+  echo "Building $FORMAT ($arch) package for $PKG from $PKG_DIR"
 
   # Package the sdist in a virtualenv
   echo "Creating virtualenv..."
@@ -634,6 +680,10 @@ fpm_build_virtualenv () {
 
   declare -a COMMAND_ARR=("fpm" "-s" "dir" "-t" "$FORMAT")
 
+  if [[ "${arch}" != "amd64" ]]; then
+    COMMAND_ARR+=("-a${arch}")
+  fi
+
   if [[ "$MAINTAINER" != "" ]]; then
     COMMAND_ARR+=('--maintainer' "$MAINTAINER")
   fi