13959: Merge branch 'master' into 13959-timeouts-and-logging
authorTom Clegg <tclegg@veritasgenetics.com>
Fri, 24 Aug 2018 16:06:18 +0000 (12:06 -0400)
committerTom Clegg <tclegg@veritasgenetics.com>
Fri, 24 Aug 2018 16:06:18 +0000 (12:06 -0400)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

22 files changed:
build/build.list
build/package-build-dockerfiles/Makefile
build/package-build-dockerfiles/centos7/Dockerfile
build/package-build-dockerfiles/debian8/Dockerfile
build/package-build-dockerfiles/debian9/Dockerfile
build/package-build-dockerfiles/ubuntu1404/Dockerfile
build/package-build-dockerfiles/ubuntu1604/Dockerfile
build/package-build-dockerfiles/ubuntu1804/Dockerfile [new file with mode: 0644]
build/package-test-dockerfiles/centos7/Dockerfile
build/package-test-dockerfiles/debian8/Dockerfile
build/package-test-dockerfiles/ubuntu1404/Dockerfile
build/package-test-dockerfiles/ubuntu1604/Dockerfile
build/package-test-dockerfiles/ubuntu1804/Dockerfile [new file with mode: 0644]
build/package-test-dockerfiles/ubuntu1804/etc-apt-preferences.d-arvados [new file with mode: 0644]
build/package-testing/deb-common-test-packages.sh
build/package-testing/test-packages-ubuntu1804.sh [new symlink]
build/run-build-packages-sso.sh
build/run-build-packages.sh
build/run-library.sh
build/run-tests.sh
sdk/cwl/setup.py
sdk/python/setup.py

index 5196e9c64d47a70601e1c816ce81abc7b1ced7ef..7f644f158dfba5f724e0cd995cbbabdabdba6b2d 100644 (file)
@@ -4,35 +4,35 @@
 
 #distribution(s)|name|version|iteration|type|architecture|extra fpm arguments
 debian8,debian9,centos7|python-gflags|2.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|google-api-python-client|1.6.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|google-api-python-client|1.6.2|2|python|all
 debian8,debian9,ubuntu1404,centos7|oauth2client|1.5.2|2|python|all
 debian8,debian9,ubuntu1404,centos7|pyasn1|0.1.7|2|python|all
 debian8,debian9,ubuntu1404,centos7|pyasn1-modules|0.0.5|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|rsa|3.4.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|uritemplate|3.0.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|httplib2|0.9.2|3|python|all
-debian8,debian9,centos7|ws4py|0.3.5|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rsa|3.4.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|uritemplate|3.0.0|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|httplib2|0.9.2|3|python|all
+debian8,debian9,centos7,ubuntu1404,ubuntu1604|ws4py|0.4.2|2|python|all
 debian8,debian9,centos7|pykka|1.2.1|2|python|all
 debian8,debian9,ubuntu1404,centos7|six|1.10.0|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|ciso8601|1.0.6|3|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|ciso8601|1.0.6|3|python|amd64
 debian8,debian9,centos7|pycrypto|2.6.1|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604|backports.ssl_match_hostname|3.5.0.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|llfuse|1.2|3|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804|backports.ssl_match_hostname|3.5.0.1|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|llfuse|1.2|3|python|amd64
 debian8,debian9,ubuntu1404,centos7|pycurl|7.19.5.3|3|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|pyyaml|3.12|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|rdflib|4.2.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pyyaml|3.12|2|python|amd64
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|rdflib|4.2.2|2|python|all
 debian8,debian9,ubuntu1404,centos7|shellescape|3.4.1|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|mistune|0.7.3|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|typing|3.6.4|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|avro|1.8.1|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|mistune|0.7.3|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|typing|3.6.4|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|avro|1.8.1|2|python|all
 debian8,debian9,ubuntu1404,centos7|ruamel.ordereddict|0.4.9|2|python|amd64
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|cachecontrol|0.11.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|pathlib2|2.3.2|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|scandir|1.7|2|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|docker-py|1.7.2|2|python3|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|cachecontrol|0.11.7|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|pathlib2|2.3.2|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|scandir|1.7|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|docker-py|1.7.2|2|python3|all
 debian8,debian9,centos7|six|1.10.0|2|python3|all
 debian8,debian9,ubuntu1404,centos7|requests|2.12.4|2|python3|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|websocket-client|0.37.0|2|python3|all
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|websocket-client|0.37.0|2|python3|all
 ubuntu1404|requests|2.4.3|2|python|all
 centos7|contextlib2|0.5.4|2|python|all
 centos7|isodate|0.5.4|2|python|all
@@ -42,9 +42,9 @@ centos7|pyparsing|2.1.10|2|python|all
 centos7|keepalive|0.5|2|python|all
 centos7|networkx|1.11|0|python|all
 centos7|psutil|5.0.1|0|python|all
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|subprocess32|3.5.1|2|python|all
-all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|lockfile|0.12.2|2|python|all|--epoch 1
+debian8,debian9,ubuntu1404,ubuntu1604,ubuntu1804,centos7|subprocess32|3.5.1|2|python|all
+centos7,debian8,debian9,ubuntu1404,ubuntu1604|ruamel.yaml|0.15.34|1|python|amd64|--python-setup-py-arguments --single-version-externally-managed
 all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
 all|junit-xml|1.8|3|python|all
 all|rdflib-jsonld|0.4.0|2|python|all
index ab1ade14deababdcc76abd11d02a99968ac0dac1..9dc536bf1eb919a7d334d3d18061db3dfd8c90d7 100644 (file)
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian8/generated debian9/generated ubuntu1204/generated ubuntu1404/generated ubuntu1604/generated
+all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
 
 centos7/generated: common-generated-all
        test -d centos7/generated || mkdir centos7/generated
@@ -16,10 +16,6 @@ debian9/generated: common-generated-all
        test -d debian9/generated || mkdir debian9/generated
        cp -rlt debian9/generated common-generated/*
 
-ubuntu1204/generated: common-generated-all
-       test -d ubuntu1204/generated || mkdir ubuntu1204/generated
-       cp -rlt ubuntu1204/generated common-generated/*
-
 ubuntu1404/generated: common-generated-all
        test -d ubuntu1404/generated || mkdir ubuntu1404/generated
        cp -rlt ubuntu1404/generated common-generated/*
@@ -28,6 +24,10 @@ ubuntu1604/generated: common-generated-all
        test -d ubuntu1604/generated || mkdir ubuntu1604/generated
        cp -rlt ubuntu1604/generated common-generated/*
 
+ubuntu1804/generated: common-generated-all
+       test -d ubuntu1804/generated || mkdir ubuntu1804/generated
+       cp -rlt ubuntu1804/generated common-generated/*
+
 GOTARBALL=go1.10.1.linux-amd64.tar.gz
 NODETARBALL=node-v6.11.2-linux-x64.tar.xz
 
index 3a8b03f190b420a69b673780a46d434c7dad8da1..216c5cbbad079bfc4ef7a16d5734579b8578995b 100644 (file)
@@ -9,12 +9,12 @@ MAINTAINER Ward Vandewege <ward@curoverse.com>
 RUN yum -q -y install make automake gcc gcc-c++ libyaml-devel patch readline-devel zlib-devel libffi-devel openssl-devel bzip2 libtool bison sqlite-devel rpm-build git perl-ExtUtils-MakeMaker libattr-devel nss-devel libcurl-devel which tar unzip scl-utils centos-release-scl postgresql-devel python-devel python-setuptools fuse-devel xz-libs git
 
 # Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler && \
-    /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
 # Install golang binary
 ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
index 54267d708e2cc2ce34c603bf5048cf816c31de86..fb1209027bdf75438790d8736b6ce710ef656b67 100644 (file)
@@ -11,12 +11,12 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev python-pip unzip
 
 # Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler && \
-    /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
 # Install golang binary
 ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
index 9ade5fa27232f6613fd07199a6c8a3d9f54565ca..e2da5ad2119f09f91d558ca327baad7dda5e3cf7 100644 (file)
@@ -18,7 +18,7 @@ RUN gpg --import /tmp/D39DC0E3.asc && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler && \
-    /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
 # Install golang binary
 ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
index 4ff47ff315bee92d127814348f621a54f64e789a..b7d4081ffb98e8aa2be1d241d30938d7eaf21820 100644 (file)
@@ -11,12 +11,12 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip 
 
 # Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler && \
-    /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
 # Install golang binary
 ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
index 7e5701f871cb987dc581fe843b1b2f3c4a2d3b7c..6220652e56e15faa8c917c8838b024e12bbe740c 100644 (file)
@@ -11,12 +11,12 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev libgnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
 
 # Install RVM
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
     /usr/local/rvm/bin/rvm-exec default gem install bundler && \
-    /usr/local/rvm/bin/rvm-exec default gem install cure-fpm --version 1.6.0b
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
 
 # Install golang binary
 ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile
new file mode 100644 (file)
index 0000000..68df124
--- /dev/null
@@ -0,0 +1,35 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM ubuntu:bionic
+MAINTAINER Ward Vandewege <ward@curoverse.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install dependencies.
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python2.7-dev python3 python-setuptools python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip tzdata
+
+# Install RVM
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    curl -L https://get.rvm.io | bash -s stable && \
+    /usr/local/rvm/bin/rvm install 2.3 && \
+    /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
+    /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+    /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
+
+# Install golang binary
+ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+RUN ln -s /usr/local/go/bin/go /usr/local/bin/
+
+# Install nodejs and npm
+ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
+RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
+
+# Old versions of setuptools cannot build a schema-salad package.
+RUN pip install --upgrade setuptools
+
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
+ENV WORKSPACE /arvados
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
index fd2f9e3d8cdbafced2990875f56ce72a2719672f..fa959a1eb8cc2e7aa39d89338ae203c5fe599933 100644 (file)
@@ -9,7 +9,7 @@ RUN yum -q -y install scl-utils centos-release-scl which tar
 
 # Install RVM
 RUN touch /var/lib/rpm/* && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
index dcf581a1e036876cb85676e305b78eb399381448..c40ed820790a8672e63b9f01e19ba4ea86c3313f 100644 (file)
@@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install RVM
 RUN apt-get update && \
     apt-get -y install --no-install-recommends curl ca-certificates && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3
index a1bc48443ea4eb48e4939d04deb8952c67220882..6d929e84948e3eb3d91f974b5b292b18b22fea62 100644 (file)
@@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install dependencies and RVM
 RUN apt-get update && \
     apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates  && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3
index 25d49dc5019112cc7d9cc6b251904780471faa79..54b1f401cf00cd2a3b6248676de51aee99915b29 100644 (file)
@@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND noninteractive
 # Install RVM
 RUN apt-get update && \
     apt-get -y install --no-install-recommends curl ca-certificates && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
     curl -L https://get.rvm.io | bash -s stable && \
     /usr/local/rvm/bin/rvm install 2.3 && \
     /usr/local/rvm/bin/rvm alias create default ruby-2.3
diff --git a/build/package-test-dockerfiles/ubuntu1804/Dockerfile b/build/package-test-dockerfiles/ubuntu1804/Dockerfile
new file mode 100644 (file)
index 0000000..506abac
--- /dev/null
@@ -0,0 +1,26 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+FROM ubuntu:bionic
+MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install RVM
+RUN apt-get update && \
+    apt-get -y install --no-install-recommends curl ca-certificates gnupg2 && \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys D39DC0E3 && \
+    curl -L https://get.rvm.io | bash -s stable && \
+    /usr/local/rvm/bin/rvm install 2.3 && \
+    /usr/local/rvm/bin/rvm alias create default ruby-2.3
+
+# udev daemon can't start in a container, so don't try.
+RUN mkdir -p /etc/udev/disabled
+
+RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu1804/ /" >>/etc/apt/sources.list
+
+# Add preferences file for the Arvados packages. This pins Arvados
+# packages at priority 501, so that older python dependency versions
+# are preferred in those cases where we need them
+ADD etc-apt-preferences.d-arvados /etc/apt/preferences.d/arvados
diff --git a/build/package-test-dockerfiles/ubuntu1804/etc-apt-preferences.d-arvados b/build/package-test-dockerfiles/ubuntu1804/etc-apt-preferences.d-arvados
new file mode 100644 (file)
index 0000000..9e24695
--- /dev/null
@@ -0,0 +1,3 @@
+Package: *
+Pin: release o=Arvados
+Pin-Priority: 501
index 900b091959107ace13d606f0a433d5d90c56f0d9..b5325224ee170bbc6170babfa8c2b95665d2057a 100755 (executable)
@@ -27,7 +27,7 @@ cd /tmp/opts
 
 export ARV_PACKAGES_DIR="/arvados/packages/$target"
 
-if [[ -f $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1) ]] ; then
+if [[ -f $(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb 2>/dev/null | head -n1) ]] ; then
     debpkg=$(ls -t "$ARV_PACKAGES_DIR/$1"_*.deb | head -n1)
 else
     debpkg=$(ls -t "$ARV_PACKAGES_DIR/processed/$1"_*.deb | head -n1)
diff --git a/build/package-testing/test-packages-ubuntu1804.sh b/build/package-testing/test-packages-ubuntu1804.sh
new file mode 120000 (symlink)
index 0000000..54ce94c
--- /dev/null
@@ -0,0 +1 @@
+deb-common-test-packages.sh
\ No newline at end of file
index bd3975841bf40cbb674dce29b43960ac1a08a524..d6a21178f8cfe3d224a08f955ca35eeb996d0dff 100755 (executable)
@@ -71,22 +71,13 @@ if [[ "$DEBUG" != 0 ]]; then
 fi
 
 case "$TARGET" in
-    debian8)
+    debian*)
         FORMAT=deb
         ;;
-    debian9)
+    ubuntu*)
         FORMAT=deb
         ;;
-    ubuntu1204)
-        FORMAT=deb
-        ;;
-    ubuntu1404)
-        FORMAT=deb
-        ;;
-    ubuntu1604)
-        FORMAT=deb
-        ;;
-    centos7)
+    centos*)
         FORMAT=rpm
         ;;
     *)
index caebac013d4db721af25655a3551371c35275782..9ffa20fe6492443feb5490ec4ce06309374b4d06 100755 (executable)
@@ -104,19 +104,13 @@ PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/dist-packages
 ## End Debian Python defaults.
 
 case "$TARGET" in
-    debian8)
+    debian*)
         FORMAT=deb
         ;;
-    debian9)
+    ubuntu*)
         FORMAT=deb
         ;;
-    ubuntu1404)
-        FORMAT=deb
-        ;;
-    ubuntu1604)
-        FORMAT=deb
-        ;;
-    centos7)
+    centos*)
         FORMAT=rpm
         PYTHON2_PACKAGE=$(rpm -qf "$(which python$PYTHON2_VERSION)" --queryformat '%{NAME}\n')
         PYTHON2_PKG_PREFIX=$PYTHON2_PACKAGE
@@ -153,8 +147,13 @@ if [[ "$?" != 0 ]]; then
   exit 1
 fi
 
-EASY_INSTALL2=$(find_easy_install -$PYTHON2_VERSION "")
-EASY_INSTALL3=$(find_easy_install -$PYTHON3_VERSION 3)
+PYTHON2_FPM_INSTALLER=(--python-easyinstall "$(find_python_program easy_install-$PYTHON2_VERSION easy_install)")
+install3=$(find_python_program easy_install-$PYTHON3_VERSION easy_install3 pip-$PYTHON3_VERSION pip3)
+if [[ $install3 =~ easy_ ]]; then
+    PYTHON3_FPM_INSTALLER=(--python-easyinstall "$install3")
+else
+    PYTHON3_FPM_INSTALLER=(--python-pip "$install3")
+fi
 
 RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
 RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`"  # absolutized and normalized
index 6ee57a4be5597114bbb0521d562883c3a1850399..8ba14949d3c0847acaaa8c2fe3671a513c7668de 100755 (executable)
@@ -20,17 +20,19 @@ debug_echo () {
     echo "$@" >"$STDOUT_IF_DEBUG"
 }
 
-find_easy_install() {
-    for version_suffix in "$@"; do
-        if "easy_install$version_suffix" --version >/dev/null 2>&1; then
-            echo "easy_install$version_suffix"
+find_python_program() {
+    prog="$1"
+    shift
+    for prog in "$@"; do
+        if "$prog" --version >/dev/null 2>&1; then
+            echo "$prog"
             return 0
         fi
     done
     cat >&2 <<EOF
 $helpmessage
 
-Error: easy_install$1 (from Python setuptools module) not found
+Error: $prog (from Python setuptools module) not found
 
 EOF
     exit 1
@@ -264,7 +266,7 @@ test_package_presence() {
     # Get the list of packages from the repos
 
     if [[ "$FORMAT" == "deb" ]]; then
-      debian_distros="jessie precise stretch trusty wheezy xenial"
+      debian_distros="jessie precise stretch trusty wheezy xenial bionic"
 
       for D in ${debian_distros}; do
         if [ ${pkgname:0:3} = "lib" ]; then
@@ -391,7 +393,7 @@ fpm_build () {
           # Make sure we build with that for consistency.
           python=python2.7
           set -- "$@" --python-bin python2.7 \
-              --python-easyinstall "$EASY_INSTALL2" \
+              "${PYTHON_FPM_INSTALLER[@]}" \
               --python-package-name-prefix "$PYTHON2_PKG_PREFIX" \
               --prefix "$PYTHON2_PREFIX" \
               --python-install-lib "$PYTHON2_INSTALL_LIB" \
@@ -407,7 +409,7 @@ fpm_build () {
           PACKAGE_TYPE=python
           python=python3
           set -- "$@" --python-bin python3 \
-              --python-easyinstall "$EASY_INSTALL3" \
+              "${PYTHON3_FPM_INSTALLER[@]}" \
               --python-package-name-prefix "$PYTHON3_PKG_PREFIX" \
               --prefix "$PYTHON3_PREFIX" \
               --python-install-lib "$PYTHON3_INSTALL_LIB" \
index e669e326c512b8d29694c05106ad26d05507a4fa..4ddbf89c1d7ccb286fcfe887fb941734bffbb519 100755 (executable)
@@ -270,6 +270,7 @@ rotate_logfile() {
 
 declare -a failures
 declare -A skip
+declare -A only
 declare -A testargs
 skip[apps/workbench_profile]=1
 # nodemanager_integration tests are not reliable, see #12061.
@@ -288,7 +289,7 @@ do
             skip[$1]=1; shift
             ;;
         --only)
-            only="$1"; skip[$1]=""; shift
+            only[$1]=1; skip[$1]=""; shift
             ;;
         --short)
             short=1
@@ -331,14 +332,19 @@ done
 # required when testing it. Skip that step if it is not needed.
 NEED_SDK_R=true
 
-if [[ ! -z "${only}" && "${only}" != "sdk/R" ]]; then
+if [[ ${#only[@]} -ne 0 ]] &&
+   [[ -z "${only['sdk/R']}" && -z "${only['doc']}" ]]; then
   NEED_SDK_R=false
 fi
 
-if [[ ! -z "${skip}" && "${skip}" == "sdk/R" ]]; then
+if [[ ${skip["sdk/R"]} == 1 && ${skip["doc"]} == 1 ]]; then
   NEED_SDK_R=false
 fi
 
+if [[ $NEED_SDK_R == false ]]; then
+       echo "R SDK not needed, it will not be installed."
+fi
+
 start_services() {
     echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...'
     if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
@@ -658,9 +664,9 @@ do_test() {
             ;;
     esac
     if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \
-              (-z "${only}" || "${only}" == "${suite}" || \
-                   "${only}" == "${1}") ||
-                  "${only}" == "${2}" ]]; then
+              (${#only[@]} -eq 0 || ${only[$suite]} -eq 1 || \
+                   ${only[$1]} -eq 1) ||
+                  ${only[$2]} -eq 1 ]]; then
         retry do_test_once ${@}
     else
         title "Skipping ${1} tests"
index 3b4dfbe99111972e638ae19ac40813d065f6d2d4..e452ce26440da32cdf3732cdc8f43f271600b432 100644 (file)
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
           'cwltool==1.0.20180806194258',
           'schema-salad==2.7.20180719125426',
           'typing >= 3.6.4',
-          'ruamel.yaml >=0.13.11, <0.15',
+          'ruamel.yaml >=0.13.11, <0.16',
           'arvados-python-client>=1.1.4.20180607143841',
           'setuptools',
           'ciso8601 >=1.0.6, <2.0.0',
index 5e066f014598560ed211a215ef8866150a77bac3..f748d52179b2d95f665027eea48dfa5c607fa0d7 100644 (file)
@@ -51,10 +51,10 @@ setup(name='arvados-python-client',
           'google-api-python-client >=1.6.2, <1.7',
           'httplib2 >=0.9.2',
           'pycurl >=7.19.5.1',
-          'ruamel.yaml >=0.13.11, <0.15',
+          'ruamel.yaml >=0.13.11, <0.16',
           'setuptools',
-          'ws4py <0.4',
-          'subprocess32>=3.5.1',
+          'ws4py >=0.4.2',
+          'subprocess32 >=3.5.1',
       ],
       test_suite='tests',
       tests_require=['pbr<1.7.0', 'mock>=1.0', 'PyYAML'],