X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2d2fcf8e83870114fe6ee20746aac362927d92aa..9ab5ebde9c712235950a1be05a37ed6bf7b7dc2f:/build/run-build-packages-one-target.sh diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index 0c474d00fd..be97ef0d13 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -32,6 +32,8 @@ Syntax: Version to build (default: \$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or 0.1.timestamp.commithash) +--skip-docker-build + Don't try to build Docker images WORKSPACE=path Path to the Arvados source tree to build packages from @@ -56,14 +58,13 @@ if ! [[ -d "$WORKSPACE" ]]; then fi PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version: \ + help,debug,test-packages,target:,command:,only-test:,force-test,only-build:,force-build,arch:,build-version:,skip-docker-build \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 fi TARGET=debian10 -ARCH= FORCE_BUILD=0 COMMAND= DEBUG= @@ -122,6 +123,9 @@ while [ $# -gt 0 ]; do fi shift ;; + --skip-docker-build) + SKIP_DOCKER_BUILD=1 + ;; --) if [ $# -gt 1 ]; then echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help" @@ -133,23 +137,28 @@ while [ $# -gt 0 ]; do done set -e +orig_umask="$(umask)" if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'" fi if [[ -n "$test_packages" ]]; then + # Packages are built world-readable, so package indexes should be too, + # especially because since 2022 apt uses an unprivileged user `_apt` to + # retrieve everything. Ensure it has permissions to read the packages + # when mounted as a volume inside the Docker container. + chmod a+rx "$WORKSPACE" "$WORKSPACE/packages" "$WORKSPACE/packages/$TARGET" + umask 022 if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then - set +e - /usr/bin/which createrepo >/dev/null - if [[ "$?" != "0" ]]; then + CREATEREPO="$(command -v createrepo createrepo_c | tail -n1)" + if [[ -z "$CREATEREPO" ]]; then echo >&2 - echo >&2 "Error: please install createrepo. E.g. sudo apt-get install createrepo" + echo >&2 "Error: please install createrepo. E.g. sudo apt install createrepo-c" echo >&2 exit 1 fi - set -e - createrepo $WORKSPACE/packages/$TARGET + "$CREATEREPO" $WORKSPACE/packages/$TARGET fi if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then @@ -177,6 +186,7 @@ if [[ -n "$test_packages" ]]; then COMMAND="/jenkins/package-testing/test-packages-$TARGET.sh" IMAGE="arvados/package-test:$TARGET" + umask "$orig_umask" else IMAGE="arvados/build:$TARGET" if [[ "$COMMAND" != "" ]]; then @@ -186,18 +196,25 @@ fi JENKINS_DIR=$(dirname "$(readlink -e "$0")") -if [[ -n "$test_packages" ]]; then - pushd "$JENKINS_DIR/package-test-dockerfiles" - make "$TARGET/generated" -else - pushd "$JENKINS_DIR/package-build-dockerfiles" - make "$TARGET/generated" -fi +if [[ "$SKIP_DOCKER_BUILD" != 1 ]] ; then + if [[ -n "$test_packages" ]]; then + pushd "$JENKINS_DIR/package-test-dockerfiles" + make "$TARGET/generated" + else + pushd "$JENKINS_DIR/package-build-dockerfiles" + make "$TARGET/generated" + fi -echo $TARGET -cd $TARGET -time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE . -popd + GOVERSION=$(grep 'const goversion =' $WORKSPACE/lib/install/deps.go |awk -F'"' '{print $2}') + + echo $TARGET + cd $TARGET + time docker build --tag "$IMAGE" \ + --build-arg HOSTTYPE=$HOSTTYPE \ + --build-arg BRANCH=$(git rev-parse HEAD) \ + --build-arg GOVERSION=$GOVERSION --no-cache . + popd +fi if test -z "$packages" ; then packages="arvados-api-server @@ -211,13 +228,12 @@ if test -z "$packages" ; then arvados-server arvados-src arvados-sync-groups - arvados-workbench + arvados-sync-users arvados-workbench2 arvados-ws crunch-dispatch-local crunch-dispatch-slurm crunch-run - crunchstat keepproxy keepstore keep-balance @@ -227,7 +243,6 @@ if test -z "$packages" ; then keep-rsync keep-block-check keep-web - libarvados-perl libpam-arvados-go python3-cwltest python3-arvados-fuse @@ -241,16 +256,13 @@ FINAL_EXITCODE=0 package_fails="" -mkdir -p "$WORKSPACE/apps/workbench/vendor/cache-$TARGET" mkdir -p "$WORKSPACE/services/api/vendor/cache-$TARGET" docker_volume_args=( -v "$JENKINS_DIR:/jenkins" -v "$WORKSPACE:/arvados" -v /arvados/services/api/vendor/bundle - -v /arvados/apps/workbench/vendor/bundle -v "$WORKSPACE/services/api/vendor/cache-$TARGET:/arvados/services/api/vendor/cache" - -v "$WORKSPACE/apps/workbench/vendor/cache-$TARGET:/arvados/apps/workbench/vendor/cache" ) if [[ -n "$test_packages" ]]; then @@ -308,16 +320,24 @@ else set +e mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null set -e + # give bundle (almost) all the cores. See also the MAKE env var that is passed into the + # docker run command below. + # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ + tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX) + cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) + printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile # Build packages. if docker run \ --rm \ "${docker_volume_args[@]}" \ + -v $tmpfile:/root/.bundle/config \ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \ --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "ONLY_BUILD=$ONLY_BUILD" \ --env "FORCE_BUILD=$FORCE_BUILD" \ --env "ARCH=$ARCH" \ + --env "MAKE=make --jobs $cores" \ "$IMAGE" $COMMAND then echo @@ -326,6 +346,8 @@ else FINAL_EXITCODE=$? echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2 fi + # Clean up the bundle config file + rm -f $tmpfile fi if test -n "$package_fails" ; then