#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -rlt centos7/generated common-generated/*
-debian8/generated: common-generated-all
- test -d debian8/generated || mkdir debian8/generated
- cp -rlt debian8/generated common-generated/*
-
debian9/generated: common-generated-all
test -d debian9/generated || mkdir debian9/generated
cp -rlt debian9/generated common-generated/*
-ubuntu1404/generated: common-generated-all
- test -d ubuntu1404/generated || mkdir ubuntu1404/generated
- cp -rlt ubuntu1404/generated common-generated/*
-
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
cp -rlt ubuntu1604/generated common-generated/*
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -rlt centos7/generated common-generated/*
-debian8/generated: common-generated-all
- test -d debian8/generated || mkdir debian8/generated
- cp -rlt debian8/generated common-generated/*
-
debian9/generated: common-generated-all
test -d debian9/generated || mkdir debian9/generated
cp -rlt debian9/generated common-generated/*
-ubuntu1404/generated: common-generated-all
- test -d ubuntu1404/generated || mkdir ubuntu1404/generated
- cp -rlt ubuntu1404/generated common-generated/*
-
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
cp -rlt ubuntu1604/generated common-generated/*
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM debian:8
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-RUN perl -ni~ -e 'print unless /jessie-updates/' /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM ubuntu:trusty
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
fi
echo
echo "START: $p test on $IMAGE" >&2
- # ulimit option can be removed when debian8 and ubuntu1404 are retired
- if docker run --ulimit nofile=4096:4096 \
+ if docker run \
--rm \
"${docker_volume_args[@]}" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
set -x
- # Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired
- if docker run --ulimit nofile=4096:4096 \
+ # Build packages.
+ if docker run \
--rm \
"${docker_volume_args[@]}" \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--debug
Output debug information (default: false)
--target <target>
- Distribution to build packages for (default: debian8)
+ Distribution to build packages for (default: debian9)
--only-build <package>
Build only a specific package (or $ONLY_BUILD from environment)
--command
DEBUG=${ARVADOS_DEBUG:-0}
EXITCODE=0
-TARGET=debian8
+TARGET=debian9
COMMAND=
PARSEDOPTS=$(getopt --name "$0" --longoptions \
if [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian8]=jessie
dd[debian9]=stretch
dd[debian10]=buster
- dd[ubuntu1404]=trusty
dd[ubuntu1604]=xenial
dd[ubuntu1804]=bionic
D=${dd[$TARGET]}
else
"$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
fi
- # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
}
initialize() {
Change to the API server directory and use the following commands:
<pre>
-$ bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ bundle exec rake config:diff
+$ RAILS_ENV=production bundle exec rake config:diff
</pre>
This command will also report if no migrations are required.
Change to the workbench server directory and use the following commands:
<pre>
-$ bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ bundle exec rake config:diff
+$ RAILS_ENV=production bundle exec rake config:diff
</pre>
This command will also report if no migrations are required.
h3. contents
-Retrieve a list of items owned by the group.
+Retrieve a list of items owned by the group. Use "recursive" to list objects within subprojects as well.
Arguments:
|order|array|Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order. Sort within a resource type by prefixing the attribute with the resource name and a period.|query|@["collections.modified_at desc"]@|
|filters|array|Conditions for filtering items.|query|@[["uuid", "is_a", "arvados#job"]]@|
|recursive|boolean (default false)|Include items owned by subprojects.|query|@true@|
-|exclude_home_project|boolean (default false)|Only return items which are visible to the user but not accessible within the user's home project. Use this to get a list of items that are shared with the user.|query|@true@|
+|exclude_home_project|boolean (default false)|Only return items which are visible to the user but not accessible within the user's home project. Use this to get a list of items that are shared with the user. Uses the logic described under the "shared" endpoint.|query|@true@|
+|include|string|If provided with the value "owner_uuid", this will return owner objects in the "included" field of the response.|query||
-Note: Because adding access tokens to manifests can be computationally expensive, the @manifest_text@ field is not included in listed collections. If you need it, request a "list of collections":{{site.baseurl}}/api/methods/collections.html with the filter @["owner_uuid", "=", GROUP_UUID]@, and @"manifest_text"@ listed in the select parameter.
+Notes:
+
+Because adding access tokens to manifests can be computationally expensive, the @manifest_text@ field is not included in listed collections. If you need it, request a "list of collections":{{site.baseurl}}/api/methods/collections.html with the filter @["owner_uuid", "=", GROUP_UUID]@, and @"manifest_text"@ listed in the select parameter.
+
+Use filters with the attribute format @<item type>.<field name>@ to filter items of a specific type. For example: @["container_requests.state", "=", "Final"]@ to filter @container_requests@ where @state@ is @Final@. All other types of items owned by this group will be unimpacted by this filter and will still be included.
+
+When called with “include=owner_uuid”, the @included@ field of the response is populated with users, projects, or other groups that own the objects returned in @items@. This can be used to fetch an object and its parent with a single API call.
-Note: Use filters with the attribute format @<item type>.<field name>@ to filter items of a specific type. For example: @["pipeline_instances.state", "=", "Complete"]@ to filter @pipeline_instances@ where @state@ is @Complete@. All other types of items owned by this group will be unimpacted by this filter and will still be included.
h3. create
h3. shared
-This endpoint returns the toplevel set of groups to which access is granted through a chain of one or more permission links rather than through direct ownership by the current user account. This is useful for clients which wish to browse the list of projects the user has permission to read which are not part of the "home" project tree.
-
-When called with "include=owner_uuid" this also returns (in the "included" field) the objects that own those projects (users or non-project groups).
+This endpoint returns the toplevel set of groups to which access is granted through a chain of one or more permission links rather than through direct ownership by the current user account. This is useful for clients which wish to browse the list of projects the user has permission to read which are not part of the "home" project tree. Similar behavior is also available with the @exclude_home_project@ option of the "contents" endpoint.
Specifically, the logic is:
the owner_uuid is a group but group_class is not a project)
</pre>
-In addition to the "include" parameter this endpoint also supports the same parameters as the "list method.":{{site.baseurl}}/api/methods.html#index
-
table(table table-bordered table-condensed).
|_. Argument |_. Type |_. Description |_. Location |_. Example |
-|include|string|If provided with the value "owner_uuid", this will return owner objects in the "included" field of the response.|query|?include=owner_uuid|
+|include|string|If provided with the value "owner_uuid", this will return owner objects in the @included@ field of the response.|query||
+
+Notes:
+
+When called with “include=owner_uuid”, the @included@ field of the response is populated with users and non-project groups that own the objects returned in @items@.
+
+In addition to the "include" parameter this endpoint also supports the same parameters as the "list method.":{{site.baseurl}}/api/methods.html#index
table(table table-bordered table-condensed).
|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
-|Debian 8 ("jessie")|Supported|Latest|
|Debian 9 ("stretch")|Supported|Latest|
-|Ubuntu 14.04 ("trusty")|Supported|Latest|
|Ubuntu 16.04 ("xenial")|Supported|Latest|
|Ubuntu 18.04 ("bionic")|Supported|Latest|
+|Ubuntu 14.04 ("trusty")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
+|Debian 8 ("jessie")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
|Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
|Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
|CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
+Arvados packages are published for current Debian releases (until the EOL date), current Ubuntu LTS releases (until the end of standard support), and the latest version of CentOS.
+
h2(#repos). Arvados package repositories
On any host where you install Arvados software, you'll need to set up an Arvados package repository. They're available for several popular distributions.
h3. Debian and Ubuntu
-Packages are available for Debian 8 ("jessie"), Debian 9 ("stretch"), Ubuntu 14.04 ("trusty"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
+Packages are available for Debian 9 ("stretch"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
First, register the Curoverse signing key in apt's database:
table(table table-bordered table-condensed).
|_. OS version|_. Command|
-|Debian 8 ("jessie")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ jessie main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 14.04 ("trusty")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ trusty main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
# ssh_key = path
# The GCE image name and network zone name to use when creating new nodes.
-image = debian-7
+image = debian
# network = your_network_name
# JSON string of service account authorizations for this cluster.
h3. 7. Set Docker image, base command, and input port for "sort" tool
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:8@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
!(screenshot)c6.png!
<pre>
requirements:
DockerRequirement:
- dockerPull: "debian:8"
+ dockerPull: "debian:9"
arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
</pre>
<notextile>
<pre><code>root@fbf1d0f529d5:/# apt-get update
-Hit http://security.debian.org jessie/updates InRelease
-Ign http://httpredir.debian.org jessie InRelease
-Ign http://apt.arvados.org jessie InRelease
-Hit http://apt.arvados.org jessie Release.gpg
-Get:1 http://security.debian.org jessie/updates/main amd64 Packages [431 kB]
-Hit http://apt.arvados.org jessie Release
-Hit http://httpredir.debian.org jessie-updates InRelease
-Get:2 http://apt.arvados.org jessie/main amd64 Packages [257 kB]
-Get:3 http://httpredir.debian.org jessie-updates/main amd64 Packages [17.6 kB]
-Hit http://httpredir.debian.org jessie Release.gpg
-Hit http://httpredir.debian.org jessie Release
-Get:4 http://httpredir.debian.org jessie/main amd64 Packages [9049 kB]
-Fetched 9755 kB in 2s (3894 kB/s)
+Get:2 http://apt.arvados.org stretch-dev InRelease [3260 B]
+Get:1 http://security-cdn.debian.org/debian-security stretch/updates InRelease [94.3 kB]
+Ign:3 http://cdn-fastly.deb.debian.org/debian stretch InRelease
+Get:4 http://cdn-fastly.deb.debian.org/debian stretch-updates InRelease [91.0 kB]
+Get:5 http://apt.arvados.org stretch-dev/main amd64 Packages [208 kB]
+Get:6 http://cdn-fastly.deb.debian.org/debian stretch Release [118 kB]
+Get:7 http://security-cdn.debian.org/debian-security stretch/updates/main amd64 Packages [499 kB]
+Get:8 http://cdn-fastly.deb.debian.org/debian stretch Release.gpg [2434 B]
+Get:9 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages.diff/Index [10.6 kB]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Fetched 1026 kB in 0s (1384 kB/s)
Reading package lists... Done
</code></pre>
</notextile>
Reading package lists... Done
Building dependency tree
Reading state information... Done
-The following extra packages will be installed:
- [...]
- libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
- r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
- r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
- r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
- [...]
-Suggested packages:
- [...]
-The following NEW packages will be installed:
- [...]
- libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
- r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
- r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
- r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
- [...]
-0 upgraded, 203 newly installed, 0 to remove and 39 not upgraded.
-Need to get 124 MB of archives.
-After this operation, 334 MB of additional disk space will be used.
-Do you want to continue [Y/n]? y
-[...]
-Get:130 http://httpredir.debian.org/debian/ jessie/main r-cran-cluster amd64 1.15.3-1 [475 kB]
-Get:131 http://httpredir.debian.org/debian/ jessie/main r-base-dev all 3.1.1-1 [4018 B]
-Get:132 http://httpredir.debian.org/debian/ jessie/main r-cran-boot all 1.3-13-1 [571 kB]
-Get:133 http://httpredir.debian.org/debian/ jessie/main r-cran-codetools all 0.2-9-1 [45.7 kB]
-Get:134 http://httpredir.debian.org/debian/ jessie/main r-cran-rpart amd64 4.1-8-1 [862 kB]
-Get:135 http://httpredir.debian.org/debian/ jessie/main r-cran-foreign amd64 0.8.61-1 [213 kB]
-[...]
-Fetched 124 MB in 52s (2380 kB/s)
-debconf: delaying package configuration, since apt-utils is not installed
-[...]
-Unpacking r-base-core (3.1.1-1+b2) ...
-Selecting previously unselected package r-base-dev.
-Preparing to unpack .../r-base-dev_3.1.1-1_all.deb ...
-Unpacking r-base-dev (3.1.1-1) ...
-Selecting previously unselected package r-cran-boot.
-Preparing to unpack .../r-cran-boot_1.3-13-1_all.deb ...
-Unpacking r-cran-boot (1.3-13-1) ...
-Selecting previously unselected package r-cran-mass.
-[...]
-Setting up r-base-core (3.1.1-1+b2) ...
-
-Creating config file /etc/R/Renviron with new version
-Setting up r-base-dev (3.1.1-1) ...
-Setting up r-cran-boot (1.3-13-1) ...
-Setting up r-cran-mass (7.3-34-1) ...
-Setting up r-cran-class (7.3-11-1) ...
+The following additional packages will be installed:
[...]
+done.
</code></pre>
</notextile>
<notextile>
<pre><code>root@fbf1d0f529d5:/# <span class="userinput">R</span>
-R version 3.1.1 (2014-07-10) -- "Sock it to Me"
-Copyright (C) 2014 The R Foundation for Statistical Computing
+R version 3.3.3 (2017-03-06) -- "Another Canoe"
+Copyright (C) 2017 The R Foundation for Statistical Computing
Platform: x86_64-pc-linux-gnu (64-bit)
R is free software and comes with ABSOLUTELY NO WARRANTY.
z1234:
API:
MaxItemsPerResponse: 1234
+ PostgreSQL:
+ Connection:
+ sslmode: require
+ Services:
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8000": {}
+ Workbench:
+ UserProfileFormFields:
+ color:
+ Type: select
+ Options:
+ fuchsia: {}
+ ApplicationMimetypesWithViewIcon:
+ whitespace: {}
`
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
ManagementToken: ""
Services:
+
+ # In each of the service sections below, the keys under
+ # InternalURLs are the endpoints where the service should be
+ # listening, and reachable from other hosts in the cluster.
+ SAMPLE:
+ InternalURLs:
+ "http://example.host:12345": {}
+ SAMPLE: {}
+ ExternalURL: "-"
+
RailsAPI:
InternalURLs: {}
ExternalURL: "-"
user: ""
password: ""
dbname: ""
+ SAMPLE: ""
API:
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Set "Required" as "true" for any of these fields to make them required.
# If any of the required fields are missing in the user's profile, the user will be
# redirected to the profile page before they can access any Workbench features.
- UserProfileFormFields: {}
+ UserProfileFormFields:
+ SAMPLE:
+ Type: select
+ FormFieldTitle: Best color
+ FormFieldDescription: your favorite color
+ Required: false
+ Position: 1
+ Options:
+ red: {}
+ blue: {}
+ green: {}
+ SAMPLE: {}
+
# exampleTextValue: # key that will be set in properties
# Type: text #
# FormFieldTitle: ""
vnd.realvnc.bed: {}
xml: {}
xsl: {}
+ SAMPLE: {}
# The maximum number of bytes to load in the log viewer
LogViewerMaxBytes: 1M
ManagementToken: ""
Services:
+
+ # In each of the service sections below, the keys under
+ # InternalURLs are the endpoints where the service should be
+ # listening, and reachable from other hosts in the cluster.
+ SAMPLE:
+ InternalURLs:
+ "http://example.host:12345": {}
+ SAMPLE: {}
+ ExternalURL: "-"
+
RailsAPI:
InternalURLs: {}
ExternalURL: "-"
user: ""
password: ""
dbname: ""
+ SAMPLE: ""
API:
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Set "Required" as "true" for any of these fields to make them required.
# If any of the required fields are missing in the user's profile, the user will be
# redirected to the profile page before they can access any Workbench features.
- UserProfileFormFields: {}
+ UserProfileFormFields:
+ SAMPLE:
+ Type: select
+ FormFieldTitle: Best color
+ FormFieldDescription: your favorite color
+ Required: false
+ Position: 1
+ Options:
+ red: {}
+ blue: {}
+ green: {}
+ SAMPLE: {}
+
# exampleTextValue: # key that will be set in properties
# Type: text #
# FormFieldTitle: ""
vnd.realvnc.bed: {}
xml: {}
xsl: {}
+ SAMPLE: {}
# The maximum number of bytes to load in the log viewer
LogViewerMaxBytes: 1M
allowed[strings.ToLower(k)] = v
}
for k, vsupp := range supplied {
+ if k == "SAMPLE" {
+ // entry will be dropped in removeSampleKeys anyway
+ continue
+ }
vexp, ok := allowed[strings.ToLower(k)]
- if !ok && expected["SAMPLE"] != nil {
+ if expected["SAMPLE"] != nil {
vexp = expected["SAMPLE"]
} else if !ok {
ldr.Logger.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
"io/ioutil"
"os"
"os/exec"
+ "reflect"
"strings"
"testing"
c.Check(logs, check.HasLen, 2)
}
+func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
+ v := reflect.Indirect(reflect.ValueOf(x))
+ switch v.Kind() {
+ case reflect.Map:
+ var stringKeys, sampleKey bool
+ iter := v.MapRange()
+ for iter.Next() {
+ k := iter.Key()
+ if k.Kind() == reflect.String {
+ stringKeys = true
+ if k.String() == "SAMPLE" || k.String() == "xxxxx" {
+ sampleKey = true
+ s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
+ }
+ }
+ }
+ if stringKeys && !sampleKey {
+ c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
+ }
+ return
+ case reflect.Struct:
+ for i := 0; i < v.NumField(); i++ {
+ val := v.Field(i)
+ if val.CanInterface() {
+ s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
+ }
+ }
+ }
+}
+
+func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
+ cfg, err := Load(bytes.NewBuffer(DefaultYAML), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ s.checkSAMPLEKeys(c, "", cfg)
+}
+
func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
var logbuf bytes.Buffer
var supplied map[string]interface{}
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- s.checkJSONErrorMatches(c, resp, `Not logged in`)
+ s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
}
func (s *FederationSuite) TestBadAuth(c *check.C) {
req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- s.checkJSONErrorMatches(c, resp, `Not logged in`)
+ s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
}
func (s *FederationSuite) TestNoAccess(c *check.C) {
req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
- s.checkJSONErrorMatches(c, resp, `.*not found`)
+ s.checkJSONErrorMatches(c, resp, `.*not found.*`)
}
func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
defer wg.Done()
err := cq.Unlock(uuid)
c.Check(err, check.NotNil)
- c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued*.")
+ c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued.*")
err = cq.Lock(uuid)
c.Check(err, check.IsNil)
UnloggedAttributes []string
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]interface{}
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
+ Value interface{}
+ Function string
+ Protected bool
+ }
PreserveVersionIfIdle Duration
TrashSweepInterval Duration
TrustAllContent bool
err = {}
end
err[:errors] ||= args
+ err[:errors].map! do |err|
+ err += " (" + Thread.current[:request_id] + ")"
+ end
err[:error_token] = [Time.now.utc.to_i, "%08x" % rand(16 ** 8)].join("+")
status = err.delete(:status) || 422
logger.error "Error #{err[:error_token]}: #{status}"
recursive: {
type: 'boolean', required: false, description: 'Include contents from child groups recursively.'
},
+ include: {
+ type: 'string', required: false, description: 'Include objects referred to by listed field in "included" (only owner_uuid)'
+ }
})
params.delete(:select)
params
token_time = token.split('+', 2).first.to_i
assert_operator(token_time, :>=, @start_stamp, "error token too old")
assert_operator(token_time, :<=, now_timestamp, "error token too new")
+ json_response['errors'].each do |err|
+ assert_match(/req-[a-z0-9]{20}/, err, "X-Request-Id value missing on error message")
+ end
end
def check_404(errmsg="Path not found")
assert_response 404
- assert_equal([errmsg], json_response['errors'])
+ json_response['errors'].each do |err|
+ assert(err.include?(errmsg), "error message '#{err}' expected to include '#{errmsg}'")
+ end
check_error_token
end
params: {specimen: {}},
headers: {'HTTP_ACCEPT' => ''})
assert_response 401
- assert_includes(json_response['errors'], "Not logged in")
+ json_response['errors'].each do |err|
+ assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+ end
end
test "login prompt respects JSON Accept header" do
params: {specimen: {}},
headers: {'HTTP_ACCEPT' => 'application/json'})
assert_response 401
- assert_includes(json_response['errors'], "Not logged in")
+ json_response['errors'].each do |err|
+ assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+ end
end
test "login prompt respects HTML Accept header" do
localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
fi
+root_cert=/var/lib/arvados/root-cert.pem
+root_cert_key=/var/lib/arvados/root-cert.key
+server_cert=/var/lib/arvados/server-cert-${localip}.pem
+server_cert_key=/var/lib/arvados/server-cert-${localip}.key
+
declare -A services
services=(
[workbench]=443
uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
-if test ! -s /var/lib/arvados/root-cert.pem ; then
+if ! openssl verify -CAfile $root_cert $root_cert ; then
# req signing request sub-command
# -new new certificate request
# -nodes "no des" don't encrypt key
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nbasicConstraints=critical,CA:true,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign")) \
- -out /var/lib/arvados/root-cert.pem \
- -keyout /var/lib/arvados/root-cert.key \
+ -out $root_cert \
+ -keyout $root_cert_key \
-days 365
- chown arvbox:arvbox /var/lib/arvados/root-cert.*
+ chown arvbox:arvbox $root_cert $root_cert_key
+ rm -f $server_cert $server_cert_key
fi
-if test ! -s /var/lib/arvados/server-cert-${localip}.pem ; then
+cp $root_cert /usr/local/share/ca-certificates/arvados-testing-cert.crt
+update-ca-certificates
+
+if ! openssl verify -CAfile $root_cert $server_cert ; then
+
+ rm -f $server_cert $server_cert_key
if [[ $localip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
san=IP:$localip
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
-out /var/lib/arvados/server-cert-${localip}.csr \
- -keyout /var/lib/arvados/server-cert-${localip}.key \
+ -keyout $server_cert_key \
-days 365
openssl x509 \
-req \
-in /var/lib/arvados/server-cert-${localip}.csr \
- -CA /var/lib/arvados/root-cert.pem \
- -CAkey /var/lib/arvados/root-cert.key \
- -out /var/lib/arvados/server-cert-${localip}.pem \
+ -CA $root_cert \
+ -CAkey $root_cert_key \
+ -out $server_cert \
-set_serial $RANDOM$RANDOM \
-extfile <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
-extensions x509_ext \
-days 365
- chown arvbox:arvbox /var/lib/arvados/server-cert-${localip}.*
+ chown arvbox:arvbox $server_cert $server_cert_key
fi
-cp /var/lib/arvados/root-cert.pem /usr/local/share/ca-certificates/arvados-testing-cert.crt
-update-ca-certificates
-
sv stop certificate
. /usr/local/lib/arvbox/common.sh
+openssl verify -CAfile $root_cert $server_cert
+
cat <<EOF >/var/lib/arvados/nginx.conf
worker_processes auto;
pid /var/lib/arvados/nginx.pid;
server {
listen *:${services[controller-ssl]} ssl default_server;
server_name controller;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://controller;
proxy_set_header Host \$http_host;
proxy_read_timeout 300s;
ssl on;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://arvados-ws;
server {
listen *:${services[workbench2-ssl]} ssl default_server;
server_name workbench2;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://workbench2;
proxy_set_header Host \$http_host;
server {
listen *:${services[keep-web-ssl]} ssl default_server;
server_name keep-web;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://keep-web;
proxy_set_header Host \$http_host;
fi
secret_token=$(cat /var/lib/arvados/sso_secret_token)
-test -s /var/lib/arvados/server-cert-${localip}.pem
+openssl verify -CAfile $root_cert $server_cert
cat >config/application.yml <<EOF
$RAILS_ENV:
fi
if test "$1" != "--only-deps" ; then
+ openssl verify -CAfile $root_cert $server_cert
exec bundle exec passenger start --port=${services[workbench]} \
--ssl --ssl-certificate=/var/lib/arvados/server-cert-${localip}.pem \
--ssl-certificate-key=/var/lib/arvados/server-cert-${localip}.key \
projects / arvados.git / commitdiff