From: Ward Vandewege Date: Fri, 9 Aug 2019 19:25:23 +0000 (-0400) Subject: 15516: Fix the arvados-server buildtime dependency for the arvados-workbench X-Git-Tag: 2.0.0~222^2~3 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/366a2efdd0ac4630f4381f3b47d70ef155ed2df4 15516: Fix the arvados-server buildtime dependency for the arvados-workbench package, and the runtime dependency for arvados-workbench and arvados-api-server. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/apps/workbench/fpm-info.sh b/apps/workbench/fpm-info.sh index 48913a14d7..22ec1ba14c 100644 --- a/apps/workbench/fpm-info.sh +++ b/apps/workbench/fpm-info.sh @@ -4,9 +4,9 @@ case "$TARGET" in centos*) - fpm_depends+=(git arvados-server) + fpm_depends+=(git) ;; debian* | ubuntu*) - fpm_depends+=(git g++ arvados-server) + fpm_depends+=(git g++) ;; esac diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile index 610dac90fe..3d3ea6f0f8 100644 --- a/build/package-build-dockerfiles/centos7/Dockerfile +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -37,10 +37,6 @@ RUN rpm -ivh epel-release-latest-7.noarch.rpm RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle -# Workbench depends on arvados-server for config manipulation -ENV GOPATH /tmp -RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados - # The version of setuptools that comes with CentOS is way too old RUN pip install --upgrade setuptools diff --git a/build/package-build-dockerfiles/debian9/Dockerfile b/build/package-build-dockerfiles/debian9/Dockerfile index f8104472b6..c403d79bcc 100644 --- a/build/package-build-dockerfiles/debian9/Dockerfile +++ b/build/package-build-dockerfiles/debian9/Dockerfile @@ -34,9 +34,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/ RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle -# Workbench depends on arvados-server for config manipulation -ENV GOPATH /tmp -RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados - ENV WORKSPACE /arvados CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"] diff --git a/build/package-build-dockerfiles/ubuntu1604/Dockerfile b/build/package-build-dockerfiles/ubuntu1604/Dockerfile index 63c119c840..90f340e66c 100644 --- a/build/package-build-dockerfiles/ubuntu1604/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1604/Dockerfile @@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/ RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle -# Workbench depends on arvados-server for config manipulation -ENV GOPATH /tmp -RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados - ENV WORKSPACE /arvados CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"] diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile index 5e4d067671..1adff74000 100644 --- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile @@ -33,9 +33,5 @@ RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/ RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle -# Workbench depends on arvados-server for config manipulation -ENV GOPATH /tmp -RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados - ENV WORKSPACE /arvados CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"] diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index 5c7dc342ee..378c9bbfa3 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -261,7 +261,6 @@ else set +e mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null set -e -set -x # Build packages. if docker run \ --rm \ diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh index ae677a12c1..8df8f981db 100755 --- a/build/run-build-packages.sh +++ b/build/run-build-packages.sh @@ -358,13 +358,16 @@ mkdir cwltest/bin && touch cwltest/bin/cwltest fpm_build_virtualenv "cwltest" "cwltest" rm -rf "$WORKSPACE/cwltest" +calculate_go_package_version arvados_server_version cmd/arvados-server +arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") + # Build the API server package test_rails_package_presence arvados-api-server "$WORKSPACE/services/api" if [[ "$?" == "0" ]]; then handle_rails_package arvados-api-server "$WORKSPACE/services/api" \ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ --description="Arvados API server - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" fi # Build the workbench server package @@ -372,6 +375,22 @@ test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" if [[ "$?" == "0" ]] ; then ( set -e + + # The workbench package has a build-time dependency on the arvados-server + # package for config manipulation, so install it first. + cd $WORKSPACE/cmd/arvados-server + get_complete_package_name arvados_server_pkgname arvados-server ${arvados_server_version} go + + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/${arvados_server_pkgname}" + if [[ ! -e ${arvados_server_pkg_path} ]]; then + arvados_server_pkg_path="$WORKSPACE/packages/$TARGET/processed/${arvados_server_pkgname}" + fi + if [[ "$FORMAT" == "deb" ]]; then + dpkg -i ${arvados_server_pkg_path} + else + rpm -i ${arvados_server_pkg_path} + fi + cd "$WORKSPACE/apps/workbench" # We need to bundle to be ready even when we build a package without vendor directory @@ -404,7 +423,7 @@ if [[ "$?" == "0" ]] ; then handle_rails_package arvados-workbench "$WORKSPACE/apps/workbench" \ "$WORKSPACE/agpl-3.0.txt" --url="https://arvados.org" \ --description="Arvados Workbench - Arvados is a free and open source platform for big data science." \ - --license="GNU Affero General Public License, version 3.0" + --license="GNU Affero General Public License, version 3.0" --depends "arvados-server = ${arvados_server_version}-${arvados_server_iteration}" fi fi diff --git a/build/run-library.sh b/build/run-library.sh index b4c2d1ddc8..af95cdc601 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -103,6 +103,37 @@ handle_ruby_gem() { fi } +calculate_go_package_version() { + local -n __returnvar="$1"; shift + local src_path="$1"; shift + + mkdir -p "$GOPATH/src/git.curoverse.com" + ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" + (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v) + + cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path" + local version="$(version_from_git)" + local timestamp="$(timestamp_from_git)" + + # Update the version number and build a new package if the vendor + # bundle has changed, or the command imports anything from the + # Arvados SDK and the SDK has changed. + declare -a checkdirs=(vendor) + if grep -qr git.curoverse.com/arvados .; then + checkdirs+=(sdk/go lib) + fi + for dir in ${checkdirs[@]}; do + cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir" + ts="$(timestamp_from_git)" + if [[ "$ts" -gt "$timestamp" ]]; then + version=$(version_from_git) + timestamp="$ts" + fi + done + + __returnvar="$version" +} + # Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" package_go_binary() { local src_path="$1"; shift @@ -110,46 +141,27 @@ package_go_binary() { local description="$1"; shift local license_file="${1:-agpl-3.0.txt}"; shift - if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]] ; then + if [[ -n "$ONLY_BUILD" ]] && [[ "$prog" != "$ONLY_BUILD" ]]; then + # arvados-workbench depends on arvados-server at build time, so even when + # only arvados-workbench is being built, we need to build arvados-server too + if [[ "$prog" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then return 0 + fi fi debug_echo "package_go_binary $src_path as $prog" local basename="${src_path##*/}" - - mkdir -p "$GOPATH/src/git.curoverse.com" - ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" - (cd "$GOPATH/src/git.curoverse.com/arvados.git" && "$GOPATH/bin/govendor" sync -v) - - cd "$GOPATH/src/git.curoverse.com/arvados.git/$src_path" - local version="$(version_from_git)" - local timestamp="$(timestamp_from_git)" - - # Update the version number and build a new package if the vendor - # bundle has changed, or the command imports anything from the - # Arvados SDK and the SDK has changed. - declare -a checkdirs=(vendor) - if grep -qr git.curoverse.com/arvados .; then - checkdirs+=(sdk/go lib) - fi - for dir in ${checkdirs[@]}; do - cd "$GOPATH/src/git.curoverse.com/arvados.git/$dir" - ts="$(timestamp_from_git)" - if [[ "$ts" -gt "$timestamp" ]]; then - version=$(version_from_git) - timestamp="$ts" - fi - done + calculate_go_package_version go_package_version $src_path cd $WORKSPACE/packages/$TARGET - test_package_presence $prog $version go + test_package_presence $prog $go_package_version go if [[ "$?" != "0" ]]; then return 1 fi - go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path" + go get -ldflags "-X main.version=${go_package_version}" "git.curoverse.com/arvados.git/$src_path" local -a switches=() systemd_unit="$WORKSPACE/${src_path}/${prog}.service" @@ -161,7 +173,7 @@ package_go_binary() { fi switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}") - fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}" + fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}" } default_iteration() { @@ -213,47 +225,62 @@ test_rails_package_presence() { test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION" } -test_package_presence() { - local pkgname="$1"; shift - local version="$1"; shift - local pkgtype="$1"; shift - local iteration="$1"; shift - local arch="$1"; shift +get_complete_package_name() { + local -n __returnvar="$1"; shift + local pkgname="$1"; shift + local version="$1"; shift + local pkgtype="$1"; shift + local iteration="$1"; shift + local arch="$1"; shift + if [[ "$iteration" == "" ]]; then + iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")" + fi - if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then - return 1 - fi + if [[ "$arch" == "" ]]; then + rpm_architecture="x86_64" + deb_architecture="amd64" - if [[ "$iteration" == "" ]]; then - iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")" + if [[ "$pkgtype" =~ ^(src)$ ]]; then + rpm_architecture="noarch" + deb_architecture="all" fi - if [[ "$arch" == "" ]]; then + # These python packages have binary components + if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then rpm_architecture="x86_64" deb_architecture="amd64" + fi + else + rpm_architecture=$arch + deb_architecture=$arch + fi - if [[ "$pkgtype" =~ ^(src)$ ]]; then - rpm_architecture="noarch" - deb_architecture="all" - fi + local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb" + if [[ "$FORMAT" == "rpm" ]]; then + # rpm packages get iteration 1 if we don't supply one + iteration=${iteration:-1} + complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm" + fi + __returnvar=${complete_pkgname} +} - # These python packages have binary components - if [[ "$pkgname" =~ (ruamel|ciso|pycrypto|pyyaml) ]]; then - rpm_architecture="x86_64" - deb_architecture="amd64" +# Test if the package already exists, if not return 0, if it does return 1 +test_package_presence() { + local pkgname="$1"; shift + local version="$1"; shift + local pkgtype="$1"; shift + local iteration="$1"; shift + local arch="$1"; shift + if [[ -n "$ONLY_BUILD" ]] && [[ "$pkgname" != "$ONLY_BUILD" ]] ; then + # arvados-workbench depends on arvados-server at build time, so even when + # only arvados-workbench is being built, we need to build arvados-server too + if [[ "$pkgname" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then + return 1 fi - else - rpm_architecture=$arch - deb_architecture=$arch fi - if [[ "$FORMAT" == "deb" ]]; then - local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb" - else - # rpm packages get iteration 1 if we don't supply one - iteration=${iteration:-1} - local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm" - fi + local full_pkgname + get_complete_package_name full_pkgname $pkgname $version $pkgtype $iteration $arch # See if we can skip building the package, only if it already exists in the # processed/ directory. If so, move it back to the packages directory to make @@ -274,32 +301,32 @@ test_package_presence() { fi repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}/main/${repo_subdir}/) - echo ${repo_pkg_list} |grep -q ${complete_pkgname} + echo ${repo_pkg_list} |grep -q ${full_pkgname} if [ $? -eq 0 ] ; then - echo "Package $complete_pkgname exists, not rebuilding!" - curl -s -o ./${complete_pkgname} http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${complete_pkgname} + echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!" + curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}/main/${repo_subdir}/${full_pkgname} return 1 - elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then - echo "Package $complete_pkgname exists, not rebuilding!" + elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then + echo "Package $full_pkgname exists, not rebuilding!" return 1 else - echo "Package $complete_pkgname not found, building" + echo "Package $full_pkgname not found, building" return 0 fi else centos_repo="http://rpm.arvados.org/CentOS/7/dev/x86_64/" repo_pkg_list=$(curl -s -o - ${centos_repo}) - echo ${repo_pkg_list} |grep -q ${complete_pkgname} + echo ${repo_pkg_list} |grep -q ${full_pkgname} if [ $? -eq 0 ]; then - echo "Package $complete_pkgname exists, not rebuilding!" - curl -s -o ./${complete_pkgname} ${centos_repo}${complete_pkgname} + echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!" + curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" ${centos_repo}${full_pkgname} return 1 - elif test -f "$WORKSPACE/packages/$TARGET/processed/${complete_pkgname}" ; then - echo "Package $complete_pkgname exists, not rebuilding!" + elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then + echo "Package $full_pkgname exists, not rebuilding!" return 1 else - echo "Package $complete_pkgname not found, building" + echo "Package $full_pkgname not found, building" return 0 fi fi @@ -681,7 +708,11 @@ fpm_build () { shift if [[ -n "$ONLY_BUILD" ]] && [[ "$PACKAGE_NAME" != "$ONLY_BUILD" ]] && [[ "$PACKAGE" != "$ONLY_BUILD" ]] ; then + # arvados-workbench depends on arvados-server at build time, so even when + # only arvados-workbench is being built, we need to build arvados-server too + if [[ "$PACKAGE_NAME" != "arvados-server" ]] || [[ "$ONLY_BUILD" != "arvados-workbench" ]]; then return 0 + fi fi local default_iteration_value="$(default_iteration "$PACKAGE" "$VERSION" "$PACKAGE_TYPE")" diff --git a/services/api/fpm-info.sh b/services/api/fpm-info.sh index 47e1e4bb17..1d5891ed62 100644 --- a/services/api/fpm-info.sh +++ b/services/api/fpm-info.sh @@ -6,9 +6,9 @@ fpm_depends+=('git >= 1.7.10') case "$TARGET" in centos*) - fpm_depends+=(libcurl-devel postgresql-devel arvados-server) + fpm_depends+=(libcurl-devel postgresql-devel) ;; debian* | ubuntu*) - fpm_depends+=(libcurl-ssl-dev libpq-dev g++ arvados-server) + fpm_depends+=(libcurl-ssl-dev libpq-dev g++) ;; esac