#
# SPDX-License-Identifier: AGPL-3.0
+SHELL := '/bin/bash'
all: centos7/generated debian10/generated debian11/generated ubuntu1804/generated ubuntu2004/generated
centos7/generated: common-generated-all
test -d ubuntu2004/generated || mkdir ubuntu2004/generated
cp -f -rlt ubuntu2004/generated common-generated/*
-GOTARBALL=go1.17.1.linux-amd64.tar.gz
-NODETARBALL=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_=DOES_NOT_EXIST
+NODETARBALL_=DOES_NOT_EXIST
+GOTARBALL_x86_64=go1.17.1.linux-amd64.tar.gz
+NODETARBALL_x86_64=node-v10.23.1-linux-x64.tar.xz
+GOTARBALL_aarch64=go1.17.1.linux-arm64.tar.gz
+NODETARBALL_aarch64=node-v10.23.1-linux-arm64.tar.xz
+
+# Get the bash variable $HOSTTYPE (this requires the SHELL line above)
+HOSTTYPE=$(shell echo $${HOSTTYPE})
+
+GOTARBALL=${GOTARBALL_$(HOSTTYPE)}
+NODETARBALL=${NODETARBALL_$(HOSTTYPE)}
+
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
target=${target#./}
echo $target
cd $target
- docker build -t arvados/build:$target .
+ docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE .
cd ..
done
#
# SPDX-License-Identifier: AGPL-3.0
+ARG HOSTTYPE
+
## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries
-FROM debian:bullseye
+FROM debian:bullseye as build_x86_64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+# On x86, we want some cross-compilation support for arm64
+# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
+# We also need libpam compiled for arm64
+ONBUILD RUN /usr/bin/dpkg --add-architecture arm64
+ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
+
+FROM debian:bullseye as build_aarch64
+# Install go
+ONBUILD ADD generated/go1.17.1.linux-arm64.tar.gz /usr/local/
+ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+# Install nodejs and npm
+ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+
+FROM build_${HOSTTYPE}
+
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
+SHELL ["/bin/bash", "-c"]
# Install dependencies.
RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev equivs
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
- echo "gem: --no-document" >> /etc/gemrc && \
+ echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
-# Install golang binary
-ADD generated/go1.17.1.linux-amd64.tar.gz /usr/local/
-RUN ln -s /usr/local/go/bin/go /usr/local/bin/
-
-# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
-RUN /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
-# We also need libpam compiled for arm64
-RUN /usr/bin/dpkg --add-architecture arm64
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64
-
-# Install nodejs and npm
-ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a)
+# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/
+ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)"
-RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+RUN echo $MAKE && git clone -b 17417-switch-to-mini_racer --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \
+ cd /tmp/arvados/services/api && \
+ /usr/local/rvm/bin/rvm-exec default bundle install && \
+ cd /tmp/arvados/apps/workbench && \
+ /usr/local/rvm/bin/rvm-exec default bundle install
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian11"]
echo $TARGET
cd $TARGET
-time docker build --tag=$IMAGE .
+time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE .
popd
if test -z "$packages" ; then
fi
# Build the workbench server package
-test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
-if [[ "$?" == "0" ]] ; then
- (
- set -e
-
- # The workbench package has a build-time dependency on the arvados-server
- # package for config manipulation, so install it first.
- cd $WORKSPACE/cmd/arvados-server
- get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
-
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
- if [[ ! -e ${arvados_server_pkg_path} ]]; then
- arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
- fi
- if [[ "$FORMAT" == "deb" ]]; then
- dpkg -i ${arvados_server_pkg_path}
- else
- rpm -i ${arvados_server_pkg_path}
- fi
-
- cd "$WORKSPACE/apps/workbench"
-
- # We need to bundle to be ready even when we build a package without vendor directory
- # because asset compilation requires it.
- bundle install --system >"$STDOUT_IF_DEBUG"
-
- # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
- # and we want that in the package, so it's easier to not exclude the tmp directory
- # from the package - empty it instead.
- rm -rf tmp
- mkdir tmp
-
- # Set up an appropriate config.yml
- arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
- mkdir -p /etc/arvados/
- mv /tmp/x /etc/arvados/config.yml
- perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
-
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
- ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
-
- # Remove generated configuration files so they don't go in the package.
- rm -rf /etc/arvados/
- )
-
- if [[ "$?" != "0" ]]; then
- echo "ERROR: Asset precompilation failed"
- EXITCODE=1
- else
- handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
- "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
- --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
- --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+if [[ "$HOSTTYPE" == "x86_64" ]]; then
+ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench"
+ if [[ "$?" == "0" ]] ; then
+ (
+ set -e
+
+ # The workbench package has a build-time dependency on the arvados-server
+ # package for config manipulation, so install it first.
+ cd $WORKSPACE/cmd/arvados-server
+ get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go
+
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}"
+ if [[ ! -e ${arvados_server_pkg_path} ]]; then
+ arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}"
+ fi
+ if [[ "$FORMAT" == "deb" ]]; then
+ dpkg -i ${arvados_server_pkg_path}
+ else
+ rpm -i ${arvados_server_pkg_path}
+ fi
+
+ cd "$WORKSPACE/apps/workbench"
+
+ # We need to bundle to be ready even when we build a package without vendor directory
+ # because asset compilation requires it.
+ bundle install --system >"$STDOUT_IF_DEBUG"
+
+ # clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
+ # and we want that in the package, so it's easier to not exclude the tmp directory
+ # from the package - empty it instead.
+ rm -rf tmp
+ mkdir tmp
+
+ # Set up an appropriate config.yml
+ arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
+ mkdir -p /etc/arvados/
+ mv /tmp/x /etc/arvados/config.yml
+ perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
+
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake npm:install >"$STDOUT_IF_DEBUG"
+ ARVADOS_CONFIG=none RAILS_ENV=production RAILS_GROUPS=assets bin/rake assets:precompile >"$STDOUT_IF_DEBUG"
+
+ # Remove generated configuration files so they don't go in the package.
+ rm -rf /etc/arvados/
+ )
+
+ if [[ "$?" != "0" ]]; then
+ echo "ERROR: Asset precompilation failed"
+ EXITCODE=1
+ else
+ handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \
+ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \
+ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \
+ --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}"
+ fi
fi
+else
+ echo "Error: building the arvados-workbench package is not yet supported on on this architecture ($HOSTTYPE)."
fi
# clean up temporary GOPATH
local description="$1"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
+ # arvados-workbench depends on arvados-server at build time, so even when
+ # only arvados-workbench is being built, we need to build arvados-server too
+ if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
+ return 0
+ fi
+ fi
+
+ native_arch="amd64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ fi
+
if [[ -n "$ONLY_ARCH" ]]; then
- package_go_binary_worker "$src_path" "$prog" "$description" "$ONLY_ARCH" "$license_file"
+ if [[ "$native_arch" == "amd64" ]] || [[ "$native_arch" == "$ONLY_ARCH" ]]; then
+ package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$ONLY_ARCH" "$license_file"
+ else
+ echo "Error: no cross compilation support for Go on $native_arch yet, can not build $prog for $ONLY_ARCH"
+ fi
else
- for arch in 'amd64' 'arm64'; do
- package_go_binary_worker "$src_path" "$prog" "$description" "$arch" "$license_file"
+ archs=($native_arch)
+ if [[ "$native_arch" == "amd64" ]]; then
+ archs=('amd64' 'arm64')
+ fi
+ for arch in $archs; do
+ package_go_binary_worker "$src_path" "$prog" "$description" "$native_arch" "$arch" "$license_file"
done
fi
}
local src_path="$1"; shift
local prog="$1"; shift
local description="$1"; shift
+ local native_arch="${1:-amd64}"; shift
local arch="${1:-amd64}"; shift
local license_file="${1:-agpl-3.0.txt}"; shift
- if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then
- # arvados-workbench depends on arvados-server at build time, so even when
- # only arvados-workbench is being built, we need to build arvados-server too
- if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then
- return 0
- fi
- fi
-
debug_echo "package_go_binary $src_path as $prog"
local basename="${src_path##*/}"
calculate_go_package_version go_package_version $src_path
fi
echo "BUILDING ${arch}"
- if [[ "$arch" == "arm64" ]]; then
+ if [[ "$arch" == "arm64" ]] && [[ "$native_arch" == "amd64" ]]; then
CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
else
GOARCH=${arch} go get -ldflags "-X git.arvados.org/arvados.git/lib/cmd.version=${go_package_version} -X main.version=${go_package_version}" "git.arvados.org/arvados.git/$src_path"
local -a switches=()
binpath=$GOPATH/bin/${basename}
- if [[ "${arch}" != "amd64" ]]; then
+ if [[ "${arch}" != "${native_arch}" ]]; then
switches+=("-a${arch}")
binpath="$GOPATH/bin/linux_${arch}/${basename}"
fi
fi
if [[ "$arch" == "" ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ native_arch="amd64"
+ rpm_native_arch="x86_64"
+ if [[ "$HOSTTYPE" == "aarch64" ]]; then
+ native_arch="arm64"
+ rpm_native_arch="arm64"
+ fi
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
if [[ "$pkgtype" =~ ^(src)$ ]]; then
rpm_architecture="noarch"
# These python packages have binary components
if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then
- rpm_architecture="x86_64"
- deb_architecture="amd64"
+ rpm_architecture="$rpm_native_arch"
+ deb_architecture="$native_arch"
fi
else
rpm_architecture=$arch