From: Tom Clegg <tclegg@veritasgenetics.com>
Date: Thu, 11 Jul 2019 17:35:09 +0000 (-0400)
Subject: Merge branch '15003-real-configs-flagged-unknown'
X-Git-Tag: 2.0.0~262
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/056d1819f3d98d8526ca59d7118ff9ce2d48aa0d?hp=93cfe7c262708fb09eda5aad1839c832816d4591

Merge branch '15003-real-configs-flagged-unknown'

refs #15003

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>
---

diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile
index 0406b0ae03..6972415152 100644
--- a/build/package-build-dockerfiles/Makefile
+++ b/build/package-build-dockerfiles/Makefile
@@ -2,24 +2,16 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
 
 centos7/generated: common-generated-all
 	test -d centos7/generated || mkdir centos7/generated
 	cp -rlt centos7/generated common-generated/*
 
-debian8/generated: common-generated-all
-	test -d debian8/generated || mkdir debian8/generated
-	cp -rlt debian8/generated common-generated/*
-
 debian9/generated: common-generated-all
 	test -d debian9/generated || mkdir debian9/generated
 	cp -rlt debian9/generated common-generated/*
 
-ubuntu1404/generated: common-generated-all
-	test -d ubuntu1404/generated || mkdir ubuntu1404/generated
-	cp -rlt ubuntu1404/generated common-generated/*
-
 ubuntu1604/generated: common-generated-all
 	test -d ubuntu1604/generated || mkdir ubuntu1604/generated
 	cp -rlt ubuntu1604/generated common-generated/*
diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile
index c6d5a15fba..c7b32968ff 100644
--- a/build/package-test-dockerfiles/Makefile
+++ b/build/package-test-dockerfiles/Makefile
@@ -2,24 +2,16 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
 
 centos7/generated: common-generated-all
 	test -d centos7/generated || mkdir centos7/generated
 	cp -rlt centos7/generated common-generated/*
 
-debian8/generated: common-generated-all
-	test -d debian8/generated || mkdir debian8/generated
-	cp -rlt debian8/generated common-generated/*
-
 debian9/generated: common-generated-all
 	test -d debian9/generated || mkdir debian9/generated
 	cp -rlt debian9/generated common-generated/*
 
-ubuntu1404/generated: common-generated-all
-	test -d ubuntu1404/generated || mkdir ubuntu1404/generated
-	cp -rlt ubuntu1404/generated common-generated/*
-
 ubuntu1604/generated: common-generated-all
 	test -d ubuntu1604/generated || mkdir ubuntu1604/generated
 	cp -rlt ubuntu1604/generated common-generated/*
diff --git a/build/package-test-dockerfiles/debian8/Dockerfile b/build/package-test-dockerfiles/debian8/Dockerfile
deleted file mode 100644
index 1d3bb87f69..0000000000
--- a/build/package-test-dockerfiles/debian8/Dockerfile
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM debian:8
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-RUN perl -ni~ -e 'print unless /jessie-updates/' /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends curl ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
-    gpg --import --no-tty /tmp/pkuczynski.asc && \
-    curl -L https://get.rvm.io | bash -s stable && \
-    /usr/local/rvm/bin/rvm install 2.5 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.5
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
diff --git a/build/package-test-dockerfiles/ubuntu1404/Dockerfile b/build/package-test-dockerfiles/ubuntu1404/Dockerfile
deleted file mode 100644
index c05dbee0f3..0000000000
--- a/build/package-test-dockerfiles/ubuntu1404/Dockerfile
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM ubuntu:trusty
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
-    apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
-    gpg --import --no-tty /tmp/pkuczynski.asc && \
-    curl -L https://get.rvm.io | bash -s stable && \
-    /usr/local/rvm/bin/rvm install 2.3 && \
-    /usr/local/rvm/bin/rvm alias create default ruby-2.3
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 69defb003b..5c7dc342ee 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -232,8 +232,7 @@ if [[ -n "$test_packages" ]]; then
         fi
         echo
         echo "START: $p test on $IMAGE" >&2
-        # ulimit option can be removed when debian8 and ubuntu1404 are retired
-        if docker run --ulimit nofile=4096:4096 \
+        if docker run \
             --rm \
             "${docker_volume_args[@]}" \
             --env ARVADOS_DEBUG=$ARVADOS_DEBUG \
@@ -263,8 +262,8 @@ else
     mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
     set -e
 set -x
-    # Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired
-    if docker run --ulimit nofile=4096:4096 \
+    # Build packages.
+    if docker run \
         --rm \
         "${docker_volume_args[@]}" \
         --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 8a675492e7..ae677a12c1 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -19,7 +19,7 @@ Options:
 --debug
     Output debug information (default: false)
 --target <target>
-    Distribution to build packages for (default: debian8)
+    Distribution to build packages for (default: debian9)
 --only-build <package>
     Build only a specific package (or $ONLY_BUILD from environment)
 --command
@@ -42,7 +42,7 @@ VENDOR="Veritas Genetics, Inc."
 
 DEBUG=${ARVADOS_DEBUG:-0}
 EXITCODE=0
-TARGET=debian8
+TARGET=debian9
 COMMAND=
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
diff --git a/build/run-library.sh b/build/run-library.sh
index ea81bbf17c..b4c2d1ddc8 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -262,10 +262,8 @@ test_package_presence() {
 
     if [[ "$FORMAT" == "deb" ]]; then
       declare -A dd
-      dd[debian8]=jessie
       dd[debian9]=stretch
       dd[debian10]=buster
-      dd[ubuntu1404]=trusty
       dd[ubuntu1604]=xenial
       dd[ubuntu1804]=bionic
       D=${dd[$TARGET]}
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 3c69ae91d6..5c71ea8e35 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -575,8 +575,6 @@ setup_virtualenv() {
     else
         "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
     fi
-    # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
-    "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
 }
 
 initialize() {
diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid
index 62017163d2..9aeb6e8888 100644
--- a/doc/install/install-manual-prerequisites.html.textile.liquid
+++ b/doc/install/install-manual-prerequisites.html.textile.liquid
@@ -35,15 +35,17 @@ h2. Supported GNU/Linux distributions
 table(table table-bordered table-condensed).
 |_. Distribution|_. State|_. Last supported version|
 |CentOS 7|Supported|Latest|
-|Debian 8 ("jessie")|Supported|Latest|
 |Debian 9 ("stretch")|Supported|Latest|
-|Ubuntu 14.04 ("trusty")|Supported|Latest|
 |Ubuntu 16.04 ("xenial")|Supported|Latest|
 |Ubuntu 18.04 ("bionic")|Supported|Latest|
+|Ubuntu 14.04 ("trusty")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
+|Debian 8 ("jessie")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
 |Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
 |Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
 |CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
 
+Arvados packages are published for current Debian releases (until the EOL date), current Ubuntu LTS releases (until the end of standard support), and the latest version of CentOS.
+
 h2(#repos). Arvados package repositories
 
 On any host where you install Arvados software, you'll need to set up an Arvados package repository.  They're available for several popular distributions.
@@ -65,7 +67,7 @@ gpgkey=http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse
 
 h3. Debian and Ubuntu
 
-Packages are available for Debian 8 ("jessie"), Debian 9 ("stretch"), Ubuntu 14.04 ("trusty"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
+Packages are available for Debian 9 ("stretch"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
 
 First, register the Curoverse signing key in apt's database:
 
@@ -75,9 +77,7 @@ Configure apt to retrieve packages from the Arvados package repository. This com
 
 table(table table-bordered table-condensed).
 |_. OS version|_. Command|
-|Debian 8 ("jessie")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ jessie main" &#x7c; sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 |Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" &#x7c; sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 14.04 ("trusty")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ trusty main" &#x7c; sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 |Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" &#x7c; sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 |Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" &#x7c; sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
 
diff --git a/doc/install/install-nodemanager.html.textile.liquid b/doc/install/install-nodemanager.html.textile.liquid
index 770527da1f..431fc10b8e 100644
--- a/doc/install/install-nodemanager.html.textile.liquid
+++ b/doc/install/install-nodemanager.html.textile.liquid
@@ -385,7 +385,7 @@ ping_host = hostname:port
 # ssh_key = path
 
 # The GCE image name and network zone name to use when creating new nodes.
-image = debian-7
+image = debian
 # network = your_network_name
 
 # JSON string of service account authorizations for this cluster.
diff --git a/doc/user/composer/composer.html.textile.liquid b/doc/user/composer/composer.html.textile.liquid
index e8ef0b6885..400c55b976 100644
--- a/doc/user/composer/composer.html.textile.liquid
+++ b/doc/user/composer/composer.html.textile.liquid
@@ -48,7 +48,7 @@ h3. 6. Create a new Command Line Tool
 
 h3. 7. Set Docker image, base command, and input port for "sort" tool
 
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:8@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@)  You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
 
 !(screenshot)c6.png!
 
diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index d62002237a..052d4509ac 100644
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -156,6 +156,6 @@ This is an optional extension field appearing on the standard @DockerRequirement
 <pre>
 requirements:
   DockerRequirement:
-    dockerPull: "debian:8"
+    dockerPull: "debian:9"
     arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
 </pre>
diff --git a/doc/user/topics/arv-docker.html.textile.liquid b/doc/user/topics/arv-docker.html.textile.liquid
index f34c21a9d7..e9e8450268 100644
--- a/doc/user/topics/arv-docker.html.textile.liquid
+++ b/doc/user/topics/arv-docker.html.textile.liquid
@@ -64,19 +64,18 @@ Next, update the package list using @apt-get update@.
 
 <notextile>
 <pre><code>root@fbf1d0f529d5:/# apt-get update
-Hit http://security.debian.org jessie/updates InRelease
-Ign http://httpredir.debian.org jessie InRelease
-Ign http://apt.arvados.org jessie InRelease
-Hit http://apt.arvados.org jessie Release.gpg
-Get:1 http://security.debian.org jessie/updates/main amd64 Packages [431 kB]
-Hit http://apt.arvados.org jessie Release
-Hit http://httpredir.debian.org jessie-updates InRelease
-Get:2 http://apt.arvados.org jessie/main amd64 Packages [257 kB]
-Get:3 http://httpredir.debian.org jessie-updates/main amd64 Packages [17.6 kB]
-Hit http://httpredir.debian.org jessie Release.gpg
-Hit http://httpredir.debian.org jessie Release
-Get:4 http://httpredir.debian.org jessie/main amd64 Packages [9049 kB]
-Fetched 9755 kB in 2s (3894 kB/s)
+Get:2 http://apt.arvados.org stretch-dev InRelease [3260 B]
+Get:1 http://security-cdn.debian.org/debian-security stretch/updates InRelease [94.3 kB]
+Ign:3 http://cdn-fastly.deb.debian.org/debian stretch InRelease
+Get:4 http://cdn-fastly.deb.debian.org/debian stretch-updates InRelease [91.0 kB]
+Get:5 http://apt.arvados.org stretch-dev/main amd64 Packages [208 kB]
+Get:6 http://cdn-fastly.deb.debian.org/debian stretch Release [118 kB]
+Get:7 http://security-cdn.debian.org/debian-security stretch/updates/main amd64 Packages [499 kB]
+Get:8 http://cdn-fastly.deb.debian.org/debian stretch Release.gpg [2434 B]
+Get:9 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages.diff/Index [10.6 kB]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Fetched 1026 kB in 0s (1384 kB/s)
 Reading package lists... Done
 </code></pre>
 </notextile>
@@ -88,54 +87,9 @@ In this example, we will install the "R" statistical language Debian package "r-
 Reading package lists... Done
 Building dependency tree
 Reading state information... Done
-The following extra packages will be installed:
-  [...]
-  libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
-  r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
-  r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
-  r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
-  [...]
-Suggested packages:
-  [...]
-The following NEW packages will be installed:
-  [...]
-  libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
-  r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
-  r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
-  r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
-  [...]
-0 upgraded, 203 newly installed, 0 to remove and 39 not upgraded.
-Need to get 124 MB of archives.
-After this operation, 334 MB of additional disk space will be used.
-Do you want to continue [Y/n]? y
-[...]
-Get:130 http://httpredir.debian.org/debian/ jessie/main r-cran-cluster amd64 1.15.3-1 [475 kB]
-Get:131 http://httpredir.debian.org/debian/ jessie/main r-base-dev all 3.1.1-1 [4018 B]
-Get:132 http://httpredir.debian.org/debian/ jessie/main r-cran-boot all 1.3-13-1 [571 kB]
-Get:133 http://httpredir.debian.org/debian/ jessie/main r-cran-codetools all 0.2-9-1 [45.7 kB]
-Get:134 http://httpredir.debian.org/debian/ jessie/main r-cran-rpart amd64 4.1-8-1 [862 kB]
-Get:135 http://httpredir.debian.org/debian/ jessie/main r-cran-foreign amd64 0.8.61-1 [213 kB]
-[...]
-Fetched 124 MB in 52s (2380 kB/s)
-debconf: delaying package configuration, since apt-utils is not installed
-[...]
-Unpacking r-base-core (3.1.1-1+b2) ...
-Selecting previously unselected package r-base-dev.
-Preparing to unpack .../r-base-dev_3.1.1-1_all.deb ...
-Unpacking r-base-dev (3.1.1-1) ...
-Selecting previously unselected package r-cran-boot.
-Preparing to unpack .../r-cran-boot_1.3-13-1_all.deb ...
-Unpacking r-cran-boot (1.3-13-1) ...
-Selecting previously unselected package r-cran-mass.
-[...]
-Setting up r-base-core (3.1.1-1+b2) ...
-
-Creating config file /etc/R/Renviron with new version
-Setting up r-base-dev (3.1.1-1) ...
-Setting up r-cran-boot (1.3-13-1) ...
-Setting up r-cran-mass (7.3-34-1) ...
-Setting up r-cran-class (7.3-11-1) ...
+The following additional packages will be installed:
 [...]
+done.
 </code></pre>
 </notextile>
 
@@ -144,8 +98,8 @@ Now we can verify that "R" is installed:
 <notextile>
 <pre><code>root@fbf1d0f529d5:/# <span class="userinput">R</span>
 
-R version 3.1.1 (2014-07-10) -- "Sock it to Me"
-Copyright (C) 2014 The R Foundation for Statistical Computing
+R version 3.3.3 (2017-03-06) -- "Another Canoe"
+Copyright (C) 2017 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
 
 R is free software and comes with ABSOLUTELY NO WARRANTY.
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index f7735a3053..169b1f7961 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -134,7 +134,7 @@ func (s *FederationSuite) TestNoAuth(c *check.C) {
 	req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
 	resp := s.testRequest(req).Result()
 	c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
-	s.checkJSONErrorMatches(c, resp, `Not logged in`)
+	s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
 }
 
 func (s *FederationSuite) TestBadAuth(c *check.C) {
@@ -142,7 +142,7 @@ func (s *FederationSuite) TestBadAuth(c *check.C) {
 	req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 	resp := s.testRequest(req).Result()
 	c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
-	s.checkJSONErrorMatches(c, resp, `Not logged in`)
+	s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
 }
 
 func (s *FederationSuite) TestNoAccess(c *check.C) {
@@ -150,7 +150,7 @@ func (s *FederationSuite) TestNoAccess(c *check.C) {
 	req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
 	resp := s.testRequest(req).Result()
 	c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
-	s.checkJSONErrorMatches(c, resp, `.*not found`)
+	s.checkJSONErrorMatches(c, resp, `.*not found.*`)
 }
 
 func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
diff --git a/lib/dispatchcloud/container/queue_test.go b/lib/dispatchcloud/container/queue_test.go
index daf7977ad5..e817a0cc7b 100644
--- a/lib/dispatchcloud/container/queue_test.go
+++ b/lib/dispatchcloud/container/queue_test.go
@@ -74,7 +74,7 @@ func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
 			defer wg.Done()
 			err := cq.Unlock(uuid)
 			c.Check(err, check.NotNil)
-			c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued*.")
+			c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued.*")
 
 			err = cq.Lock(uuid)
 			c.Check(err, check.IsNil)
diff --git a/services/api/app/controllers/application_controller.rb b/services/api/app/controllers/application_controller.rb
index e07a5aca79..d5bc3f35d7 100644
--- a/services/api/app/controllers/application_controller.rb
+++ b/services/api/app/controllers/application_controller.rb
@@ -183,6 +183,9 @@ class ApplicationController < ActionController::Base
       err = {}
     end
     err[:errors] ||= args
+    err[:errors].map! do |err|
+      err += " (" + Thread.current[:request_id] + ")"
+    end
     err[:error_token] = [Time.now.utc.to_i, "%08x" % rand(16 ** 8)].join("+")
     status = err.delete(:status) || 422
     logger.error "Error #{err[:error_token]}: #{status}"
diff --git a/services/api/test/functional/application_controller_test.rb b/services/api/test/functional/application_controller_test.rb
index b5f71acb53..b74ff0f41d 100644
--- a/services/api/test/functional/application_controller_test.rb
+++ b/services/api/test/functional/application_controller_test.rb
@@ -24,11 +24,16 @@ class ApplicationControllerTest < ActionController::TestCase
     token_time = token.split('+', 2).first.to_i
     assert_operator(token_time, :>=, @start_stamp, "error token too old")
     assert_operator(token_time, :<=, now_timestamp, "error token too new")
+    json_response['errors'].each do |err|
+      assert_match(/req-[a-z0-9]{20}/, err, "X-Request-Id value missing on error message")
+    end
   end
 
   def check_404(errmsg="Path not found")
     assert_response 404
-    assert_equal([errmsg], json_response['errors'])
+    json_response['errors'].each do |err|
+      assert(err.include?(errmsg), "error message '#{err}' expected to include '#{errmsg}'")
+    end
     check_error_token
   end
 
diff --git a/services/api/test/integration/login_workflow_test.rb b/services/api/test/integration/login_workflow_test.rb
index 8691030e9d..f0741fcfde 100644
--- a/services/api/test/integration/login_workflow_test.rb
+++ b/services/api/test/integration/login_workflow_test.rb
@@ -10,7 +10,9 @@ class LoginWorkflowTest < ActionDispatch::IntegrationTest
       params: {specimen: {}},
       headers: {'HTTP_ACCEPT' => ''})
     assert_response 401
-    assert_includes(json_response['errors'], "Not logged in")
+    json_response['errors'].each do |err|
+      assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+    end
   end
 
   test "login prompt respects JSON Accept header" do
@@ -18,7 +20,9 @@ class LoginWorkflowTest < ActionDispatch::IntegrationTest
       params: {specimen: {}},
       headers: {'HTTP_ACCEPT' => 'application/json'})
     assert_response 401
-    assert_includes(json_response['errors'], "Not logged in")
+    json_response['errors'].each do |err|
+      assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+    end
   end
 
   test "login prompt respects HTML Accept header" do