From 37c46ca294ce95242ac424c8375a130d783f6d1e Mon Sep 17 00:00:00 2001 From: Ward Vandewege Date: Thu, 7 Feb 2019 16:06:47 -0500 Subject: [PATCH] 14819: upgrade the arvados/jobs image to Debian Stretch. Also add argument to the build script to choose which package repo to build from. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- .licenseignore | 3 +- build/run-build-docker-jobs-image.sh | 48 +++++++++++++++++------- docker/jobs/1078ECD7.key | 30 +++++++++++++++ docker/jobs/Dockerfile | 22 ++++++----- docker/jobs/apt.arvados.org-dev.list | 2 + docker/jobs/apt.arvados.org-stable.list | 2 + docker/jobs/apt.arvados.org-testing.list | 2 + docker/jobs/apt.arvados.org.list | 2 - 8 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 docker/jobs/1078ECD7.key create mode 100644 docker/jobs/apt.arvados.org-dev.list create mode 100644 docker/jobs/apt.arvados.org-stable.list create mode 100644 docker/jobs/apt.arvados.org-testing.list delete mode 100644 docker/jobs/apt.arvados.org.list diff --git a/.licenseignore b/.licenseignore index 06519a98e8..45028bf888 100644 --- a/.licenseignore +++ b/.licenseignore @@ -15,7 +15,8 @@ build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados doc/fonts/* doc/user/cwl/federated/* */docker_image -docker/jobs/apt.arvados.org.list +docker/jobs/apt.arvados.org*.list +docker/jobs/1078ECD7.key */en.bootstrap.yml *font-awesome.css *.gif diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh index 7186a22091..7d7e1fc8ab 100755 --- a/build/run-build-docker-jobs-image.sh +++ b/build/run-build-docker-jobs-image.sh @@ -5,21 +5,24 @@ function usage { echo >&2 - echo >&2 "usage: $0 [options]" + echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]" echo >&2 echo >&2 "$0 options:" echo >&2 " -t, --tags version tag for docker" + echo >&2 " -r, --repo Arvados package repot to use: dev, testing, stable (default: dev)" echo >&2 " -u, --upload Upload the images (docker push)" echo >&2 " --no-cache Don't use build cache" echo >&2 " -h, --help Display this help and exit" echo >&2 - echo >&2 " If no options are given, just builds the images." + echo >&2 " WORKSPACE=path Path to the Arvados source tree to build from" + echo >&2 } upload=false +REPO=dev # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros). -TEMP=`getopt -o hut: \ - --long help,upload,no-cache,tags: \ +TEMP=`getopt -o hut:r: \ + --long help,upload,no-cache,tags,repo: \ -n "$0" -- "$@"` if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi @@ -50,6 +53,19 @@ do ;; esac ;; + -r | --repo) + case "$2" in + "") + echo "ERROR: --repo needs a parameter"; + usage; + exit 1 + ;; + *) + REPO="$2"; + shift 2 + ;; + esac + ;; --) shift break @@ -69,6 +85,16 @@ exit_cleanly() { exit $EXITCODE } +# Sanity check +if ! [[ -n "$WORKSPACE" ]]; then + usage; + echo >&2 "Error: WORKSPACE environment variable not set" + echo >&2 + exit 1 +fi + +echo $WORKSPACE + COLUMNS=80 . $WORKSPACE/build/run-library.sh @@ -88,16 +114,6 @@ docker_push () { checkexit $ECODE "docker push $*" } -# Sanity check -if ! [[ -n "$WORKSPACE" ]]; then - echo >&2 - echo >&2 "Error: WORKSPACE environment variable not set" - echo >&2 - exit 1 -fi - -echo $WORKSPACE - # find the docker binary DOCKER=`which docker.io` @@ -153,6 +169,7 @@ cd docker/jobs docker build $NOCACHE \ --build-arg python_sdk_version=${python_sdk_version} \ --build-arg cwl_runner_version=${cwl_runner_version} \ + --build-arg repo_version=${REPO} \ -t arvados/jobs:$cwl_runner_version_orig . ECODE=$? @@ -175,6 +192,9 @@ if docker --version |grep " 1\.[0-9]\." ; then # -f flag removed in Docker 1.12 FORCE=-f fi + +#docker export arvados/jobs:$cwl_runner_version_orig | docker import - arvados/jobs:$cwl_runner_version_orig + if ! [[ -z "$version_tag" ]]; then docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag" else diff --git a/docker/jobs/1078ECD7.key b/docker/jobs/1078ECD7.key new file mode 100644 index 0000000000..edc62f48ff --- /dev/null +++ b/docker/jobs/1078ECD7.key @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBEzhgeoBCAChhoK1dqpWzNyDWqRGEvdFdkJaA9D2HRwKPfBfjAoePX6ZyrpA +ItlUsvt/8s/DRiTiPEFQR4S7VqocmU6whJc3gDEGyOM6b1NF873lIfSVwUoE42QE +a76dO8woOYgLUyxu2mKG+bJgGMumjBJt6ZOndYVjTYB/7sEeVxwmMVulfZe0s6zg +ut0+SoTYg2R36qIqeIcWllYt97sEYnyy1qXMis4/3IZnuWkS/frsPR3aeUI4W+o2 +NDN1kj49+LMe7Fb5b7jZY08rZbAWXi1rU1hQx4jC9RvYqlT4HNld4Bn7os1IvOOA +wNiR0oiVdiuDbBxcMvRPktxMrFVjowusRLq/ABEBAAG0PUN1cm92ZXJzZSwgSW5j +IEF1dG9tYXRpYyBTaWduaW5nIEtleSA8c3lzYWRtaW5AY3Vyb3ZlcnNlLmNvbT6J +ATgEEwECACIFAlNgYIECGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFcW +WREQeOzXPkEH/jQJDIYI1dxWcYiA+hczmpaZvN2/pc/kwIW/6a03+6zqmSNkebOE +TgoDILacSYc17hy20R1/rWyUstOMKcEgFDBlSehhHyl0f7q/w7d8Ais6MabzsPfx +IceJpsjUg87+BR7qWhgQ0sxmtIF2TKuTFLs+nkGsgSsiBOEF4NvHxuj3HD4y8F27 +HNqrkqwjLS8xJwwH5Gp2uMEVr1AXIH3iSRjJ8X124s8iEP97Q/3IazoYRf9/MCSm +QEx8KzxwDX6t4bW6O4D01K+e9gdkTY70dcMgJoqm5IsX7yxjEubiOunphtlJnZ9d +Oi1yBN5UM3pWKAdcfRj4rcfV9Simvpx9av+5AQ0ETOGB6gEIAMAA0HVMG0BbdnU7 +wWgl5eFdT0AUSrXK/WdcKqVEGGv+c68NETSHWZOJX7O46Eao4gY4cTYprVMBzxpY +/BtQSYLpE0HLvBc1fcFd61Yz4H/9rGSNY0GcIQEbOjbJY5mr8qFsQ1K/mAf3aUL3 +b6ni4sHVicRiRr0Gl4Ihorlskpfu1SHs/C5tvTSVNF9p4vtl5892y1yILQeVpcBs +NCR7MUpdS49xCpvnAWsDZX+ij6LTR3lzCm/ZLCg4gNuZkjgU9oqVfGkqysW7WZ8S +OLvzAwUw7i1EIFX8q6QdudGoezxz8m8OgZM1v8AFpYEKlhEPf1W0MSfaRDwrj866 +8nCLruEAEQEAAYkBHwQYAQIACQUCTOGB6gIbDAAKCRBXFlkREHjs199EB/4+p0G1 +3PHxt6rLWSCGXobDOu4ZOA/qnv0D/JhOLroFds5TzQv6vnS8eAkhCTjHVA+b58cm +kXpI0oYcD4ZP+KK1CHKq2rGfwou7HfAF+icnNqYkeBOkjjbCgkvBlcCInuAuU8JX +DZMkfFk52+eBKwTjS/J/fQp0vDru8bHLp98WgdRHWfJQ3mc3gz4A5sR6zhrGPW6/ +ssnROS4dC2Ohp35GpgN1KjD3EmEw5RoSBYlyrARCaMsivgIKMxGUEyFZWhuJt3N1 +2MTddRwz28hbmYCi+MzHYDbRv+cSyUDmvXaWhfkNKBepClBA1rTWBcldit5vvlqr +yPet6wIKrtLGhAqZ +=CLkG +-----END PGP PUBLIC KEY BLOCK----- diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile index c0fe145db1..758262f03f 100644 --- a/docker/jobs/Dockerfile +++ b/docker/jobs/Dockerfile @@ -2,24 +2,28 @@ # # SPDX-License-Identifier: Apache-2.0 -# Based on Debian Jessie -FROM debian:jessie -MAINTAINER Ward Vandewege +# Based on Debian Stretch +FROM debian:stretch +MAINTAINER Ward Vandewege ENV DEBIAN_FRONTEND noninteractive -ADD apt.arvados.org.list /etc/apt/sources.list.d/ -RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7 -RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 +RUN apt-get update -q +RUN apt-get install -yq --no-install-recommends gnupg + +ARG repo_version +RUN echo repo_version $repo_version +ADD apt.arvados.org-$repo_version.list /etc/apt/sources.list.d/ + +ADD 1078ECD7.key /tmp/ +RUN cat /tmp/1078ECD7.key | apt-key add - ARG python_sdk_version ARG cwl_runner_version RUN echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version RUN apt-get update -q -RUN apt-get install -yq --no-install-recommends \ - git python-pip python-virtualenv \ - python-dev libgnutls28-dev libcurl4-gnutls-dev nodejs \ +RUN apt-get install -yq --no-install-recommends nodejs \ python-arvados-python-client=$python_sdk_version \ python-arvados-cwl-runner=$cwl_runner_version diff --git a/docker/jobs/apt.arvados.org-dev.list b/docker/jobs/apt.arvados.org-dev.list new file mode 100644 index 0000000000..468000ed29 --- /dev/null +++ b/docker/jobs/apt.arvados.org-dev.list @@ -0,0 +1,2 @@ +# apt.arvados.org +deb http://apt.arvados.org/ stretch-dev main diff --git a/docker/jobs/apt.arvados.org-stable.list b/docker/jobs/apt.arvados.org-stable.list new file mode 100644 index 0000000000..afbc51effe --- /dev/null +++ b/docker/jobs/apt.arvados.org-stable.list @@ -0,0 +1,2 @@ +# apt.arvados.org +deb http://apt.arvados.org/ stretch main diff --git a/docker/jobs/apt.arvados.org-testing.list b/docker/jobs/apt.arvados.org-testing.list new file mode 100644 index 0000000000..c8ea91d070 --- /dev/null +++ b/docker/jobs/apt.arvados.org-testing.list @@ -0,0 +1,2 @@ +# apt.arvados.org +deb http://apt.arvados.org/ stretch-testing main diff --git a/docker/jobs/apt.arvados.org.list b/docker/jobs/apt.arvados.org.list deleted file mode 100644 index 11b98e25bb..0000000000 --- a/docker/jobs/apt.arvados.org.list +++ /dev/null @@ -1,2 +0,0 @@ -# apt.arvados.org -deb http://apt.arvados.org/ jessie-dev main -- 2.30.2