case "$TARGET" in
centos*)
- fpm_depends+=(git)
+ fpm_depends+=(git arvados-server)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++)
+ fpm_depends+=(git g++ arvados-server)
;;
esac
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -rlt centos7/generated common-generated/*
-debian8/generated: common-generated-all
- test -d debian8/generated || mkdir debian8/generated
- cp -rlt debian8/generated common-generated/*
-
debian9/generated: common-generated-all
test -d debian9/generated || mkdir debian9/generated
cp -rlt debian9/generated common-generated/*
-ubuntu1404/generated: common-generated-all
- test -d ubuntu1404/generated || mkdir ubuntu1404/generated
- cp -rlt ubuntu1404/generated common-generated/*
-
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
cp -rlt ubuntu1604/generated common-generated/*
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
cp -rlt ubuntu1804/generated common-generated/*
-GOTARBALL=go1.10.1.linux-amd64.tar.gz
+GOTARBALL=go1.12.7.linux-amd64.tar.gz
NODETARBALL=node-v6.11.2-linux-x64.tar.xz
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
common-generated-all: common-generated/$(GOTARBALL) common-generated/$(NODETARBALL) common-generated/$(RVMKEY1) common-generated/$(RVMKEY2)
common-generated/$(GOTARBALL): common-generated
- wget -cqO common-generated/$(GOTARBALL) http://storage.googleapis.com/golang/$(GOTARBALL)
+ wget -cqO common-generated/$(GOTARBALL) https://dl.google.com/go/$(GOTARBALL)
common-generated/$(NODETARBALL): common-generated
wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v6.11.2/$(NODETARBALL)
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
RUN rpm -ivh epel-release-latest-7.noarch.rpm
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
# The version of setuptools that comes with CentOS is way too old
RUN pip install --upgrade setuptools
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
-ADD generated/go1.10.1.linux-amd64.tar.gz /usr/local/
+ADD generated/go1.12.7.linux-amd64.tar.gz /usr/local/
RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
ADD generated/node-v6.11.2-linux-x64.tar.xz /usr/local/
RUN ln -s /usr/local/node-v6.11.2-linux-x64/bin/* /usr/local/bin/
-RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
+
+# Workbench depends on arvados-server for config manipulation
+ENV GOPATH /tmp
+RUN mkdir -p $GOPATH/src/git.curoverse.com && ln -sT /tmp/arvados $GOPATH/src/git.curoverse.com/arvados.git && cd $GOPATH/src/git.curoverse.com/arvados.git/cmd/arvados-server && go get -v github.com/kardianos/govendor && $GOPATH/bin/govendor sync && go get && go build && cp arvados-server /usr/local/bin/ && rm -rf /tmp/arvados
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1804"]
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian8/generated debian9/generated ubuntu1404/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian9/generated ubuntu1604/generated ubuntu1804/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
cp -rlt centos7/generated common-generated/*
-debian8/generated: common-generated-all
- test -d debian8/generated || mkdir debian8/generated
- cp -rlt debian8/generated common-generated/*
-
debian9/generated: common-generated-all
test -d debian9/generated || mkdir debian9/generated
cp -rlt debian9/generated common-generated/*
-ubuntu1404/generated: common-generated-all
- test -d ubuntu1404/generated || mkdir ubuntu1404/generated
- cp -rlt ubuntu1404/generated common-generated/*
-
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
cp -rlt ubuntu1604/generated common-generated/*
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM debian:8
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-RUN perl -ni~ -e 'print unless /jessie-updates/' /etc/apt/sources.list
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/debian8/ /" >>/etc/apt/sources.list
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-FROM ubuntu:trusty
-MAINTAINER Ward Vandewege <wvandewege@veritasgenetics.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-# Install dependencies
-RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates python2.7-dev python3 python-setuptools python3-setuptools libcurl4-gnutls-dev curl git libattr1-dev libfuse-dev libpq-dev python-pip unzip binutils build-essential ca-certificates
-
-# Install RVM
-ADD generated/mpapis.asc /tmp/
-ADD generated/pkuczynski.asc /tmp/
-RUN gpg --import --no-tty /tmp/mpapis.asc && \
- gpg --import --no-tty /tmp/pkuczynski.asc && \
- curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
-
-# udev daemon can't start in a container, so don't try.
-RUN mkdir -p /etc/udev/disabled
-
-RUN echo "deb file:///arvados/packages/ubuntu1404/ /" >>/etc/apt/sources.list
fi
report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
-report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
+if printf '%s\n' "$CONFIG_PATH" | grep -Fqe "sso"; then
+ report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
+else
+ report_not_ready "$APPLICATION_READY" "/etc/arvados/config.yml"
+fi
fi
echo
echo "START: $p test on $IMAGE" >&2
- # ulimit option can be removed when debian8 and ubuntu1404 are retired
- if docker run --ulimit nofile=4096:4096 \
+ if docker run \
--rm \
"${docker_volume_args[@]}" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
set -x
- # Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired
- if docker run --ulimit nofile=4096:4096 \
+ # Build packages.
+ if docker run \
--rm \
"${docker_volume_args[@]}" \
--env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
--debug
Output debug information (default: false)
--target <target>
- Distribution to build packages for (default: debian8)
+ Distribution to build packages for (default: debian9)
--only-build <package>
Build only a specific package (or $ONLY_BUILD from environment)
--command
DEBUG=${ARVADOS_DEBUG:-0}
EXITCODE=0
-TARGET=debian8
+TARGET=debian9
COMMAND=
PARSEDOPTS=$(getopt --name "$0" --longoptions \
rm -rf tmp
mkdir tmp
- # Set up application.yml and production.rb so that asset precompilation works
- \cp config/application.yml.example config/application.yml -f
- \cp config/environments/production.rb.example config/environments/production.rb -f
- sed -i 's/secret_token: ~/secret_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/' config/application.yml
- sed -i 's/keep_web_url: false/keep_web_url: exampledotcom/' config/application.yml
+ # Set up an appropriate config.yml
+ arvados-server config-dump -config <(cat /etc/arvados/config.yml 2>/dev/null || echo "Clusters: {zzzzz: {}}") > /tmp/x
+ mkdir -p /etc/arvados/
+ mv /tmp/x /etc/arvados/config.yml
+ perl -p -i -e 'BEGIN{undef $/;} s/WebDAV(.*?):\n( *)ExternalURL: ""/WebDAV$1:\n$2ExternalURL: "example.com"/g' /etc/arvados/config.yml
RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake npm:install >/dev/null
RAILS_ENV=production RAILS_GROUPS=assets bundle exec rake assets:precompile >/dev/null
# Remove generated configuration files so they don't go in the package.
- rm config/application.yml config/environments/production.rb
+ rm -rf /etc/arvados/
)
if [[ "$?" != "0" ]]; then
if [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian8]=jessie
dd[debian9]=stretch
dd[debian10]=buster
- dd[ubuntu1404]=trusty
dd[ubuntu1604]=xenial
dd[ubuntu1804]=bionic
D=${dd[$TARGET]}
echo -n 'go: '
go version \
|| fatal "No go binary. See http://golang.org/doc/install"
- [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \
- || fatal "Go >= 1.10 required. See http://golang.org/doc/install"
+ [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \
+ || fatal "Go >= 1.12 required. See http://golang.org/doc/install"
echo -n 'gcc: '
gcc --version | egrep ^gcc \
|| fatal "No gcc. Try: apt-get install build-essential"
else
"$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
fi
- # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
}
initialize() {
Change to the API server directory and use the following commands:
<pre>
-$ bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ bundle exec rake config:diff
+$ RAILS_ENV=production bundle exec rake config:diff
</pre>
This command will also report if no migrations are required.
Change to the workbench server directory and use the following commands:
<pre>
-$ bundle exec rake config:migrate > config.yml
+$ RAILS_ENV=production bundle exec rake config:migrate > config.yml
$ cp config.yml /etc/arvados/config.yml
</pre>
If you wish to update @config.yml@ configuration by hand, or check that everything has been migrated, use @config:diff@ to print configuration items that differ between @application.yml@ and the system @config.yml@.
<pre>
-$ bundle exec rake config:diff
+$ RAILS_ENV=production bundle exec rake config:diff
</pre>
This command will also report if no migrations are required.
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+
+h4(#substringsearchfilter). Filtering using substring search
+
+Resources can also be filtered by searching for a substring in attributes of type @string@, @array of strings@, @text@, and @hash@, which are indexed in the database specifically for search. To use substring search, the filter must:
+
+* Specify @any@ as the attribute
+* Use either the @like@ or @ilike@ operator
+* Have an operand of type @string@ that is wrapped in the SQL pattern match wildcard character @%@
+
+For example, the @["any", "like", "%foo%"]@ filter will return all resources that contain @foo@ in the content of at least one attribute of the previously defined types. This is the recommended way to do keyword and file name search across the entire database. Note that only exact substring matches are returned and results are unranked and returned in the order specified by the @list@ @order@ argument.
+
h4(#subpropertyfilters). Filtering on subproperties
-Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property@. You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@. Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
+Some record types have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property@. You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@. Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
table(table table-bordered table-condensed).
|_. Operator|_. Operand type|_. Description|_. Example|
table(table table-bordered table-condensed).
|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
-|Debian 8 ("jessie")|Supported|Latest|
|Debian 9 ("stretch")|Supported|Latest|
-|Ubuntu 14.04 ("trusty")|Supported|Latest|
|Ubuntu 16.04 ("xenial")|Supported|Latest|
|Ubuntu 18.04 ("bionic")|Supported|Latest|
+|Ubuntu 14.04 ("trusty")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
+|Debian 8 ("jessie")|EOL|5f943cd451acfbdcddd84e791738c3aa5926bfed (2019-07-10)|
|Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
|Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
|CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
+Arvados packages are published for current Debian releases (until the EOL date), current Ubuntu LTS releases (until the end of standard support), and the latest version of CentOS.
+
h2(#repos). Arvados package repositories
On any host where you install Arvados software, you'll need to set up an Arvados package repository. They're available for several popular distributions.
h3. Debian and Ubuntu
-Packages are available for Debian 8 ("jessie"), Debian 9 ("stretch"), Ubuntu 14.04 ("trusty"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
+Packages are available for Debian 9 ("stretch"), Ubuntu 16.04 ("xenial") and Ubuntu 18.04 ("bionic").
First, register the Curoverse signing key in apt's database:
table(table table-bordered table-condensed).
|_. OS version|_. Command|
-|Debian 8 ("jessie")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ jessie main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 14.04 ("trusty")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ trusty main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" | sudo tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
# ssh_key = path
# The GCE image name and network zone name to use when creating new nodes.
-image = debian-7
+image = debian
# network = your_network_name
# JSON string of service account authorizations for this cluster.
@arv collection list --filters '[["name", "=", "PGP VAR inputs"], ["created_at", ">=", "2014-10-01"]]'@
-will return a list of all collections visible to the current user which are named "PGP VAR inputs" and were created on or after October 1, 2014.
+will return a list of all collections visible to the current user which are named "PGP VAR inputs" and were created on or after October 1, 2014. See the "Common resource methods":{{site.baseurl}}/api/methods.html#index page for more details on using @list@ and @--filters@.
h3. 7. Set Docker image, base command, and input port for "sort" tool
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:8@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
!(screenshot)c6.png!
<pre>
requirements:
DockerRequirement:
- dockerPull: "debian:8"
+ dockerPull: "debian:9"
arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
</pre>
<notextile>
<pre><code>root@fbf1d0f529d5:/# apt-get update
-Hit http://security.debian.org jessie/updates InRelease
-Ign http://httpredir.debian.org jessie InRelease
-Ign http://apt.arvados.org jessie InRelease
-Hit http://apt.arvados.org jessie Release.gpg
-Get:1 http://security.debian.org jessie/updates/main amd64 Packages [431 kB]
-Hit http://apt.arvados.org jessie Release
-Hit http://httpredir.debian.org jessie-updates InRelease
-Get:2 http://apt.arvados.org jessie/main amd64 Packages [257 kB]
-Get:3 http://httpredir.debian.org jessie-updates/main amd64 Packages [17.6 kB]
-Hit http://httpredir.debian.org jessie Release.gpg
-Hit http://httpredir.debian.org jessie Release
-Get:4 http://httpredir.debian.org jessie/main amd64 Packages [9049 kB]
-Fetched 9755 kB in 2s (3894 kB/s)
+Get:2 http://apt.arvados.org stretch-dev InRelease [3260 B]
+Get:1 http://security-cdn.debian.org/debian-security stretch/updates InRelease [94.3 kB]
+Ign:3 http://cdn-fastly.deb.debian.org/debian stretch InRelease
+Get:4 http://cdn-fastly.deb.debian.org/debian stretch-updates InRelease [91.0 kB]
+Get:5 http://apt.arvados.org stretch-dev/main amd64 Packages [208 kB]
+Get:6 http://cdn-fastly.deb.debian.org/debian stretch Release [118 kB]
+Get:7 http://security-cdn.debian.org/debian-security stretch/updates/main amd64 Packages [499 kB]
+Get:8 http://cdn-fastly.deb.debian.org/debian stretch Release.gpg [2434 B]
+Get:9 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages.diff/Index [10.6 kB]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Get:10 http://cdn-fastly.deb.debian.org/debian stretch-updates/main amd64 Packages 2019-07-08-0821.07.pdiff [445 B]
+Fetched 1026 kB in 0s (1384 kB/s)
Reading package lists... Done
</code></pre>
</notextile>
Reading package lists... Done
Building dependency tree
Reading state information... Done
-The following extra packages will be installed:
- [...]
- libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
- r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
- r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
- r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
- [...]
-Suggested packages:
- [...]
-The following NEW packages will be installed:
- [...]
- libxxf86vm1 make patch r-base-core r-base-dev r-cran-boot r-cran-class
- r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth
- r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet
- r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended
- [...]
-0 upgraded, 203 newly installed, 0 to remove and 39 not upgraded.
-Need to get 124 MB of archives.
-After this operation, 334 MB of additional disk space will be used.
-Do you want to continue [Y/n]? y
-[...]
-Get:130 http://httpredir.debian.org/debian/ jessie/main r-cran-cluster amd64 1.15.3-1 [475 kB]
-Get:131 http://httpredir.debian.org/debian/ jessie/main r-base-dev all 3.1.1-1 [4018 B]
-Get:132 http://httpredir.debian.org/debian/ jessie/main r-cran-boot all 1.3-13-1 [571 kB]
-Get:133 http://httpredir.debian.org/debian/ jessie/main r-cran-codetools all 0.2-9-1 [45.7 kB]
-Get:134 http://httpredir.debian.org/debian/ jessie/main r-cran-rpart amd64 4.1-8-1 [862 kB]
-Get:135 http://httpredir.debian.org/debian/ jessie/main r-cran-foreign amd64 0.8.61-1 [213 kB]
-[...]
-Fetched 124 MB in 52s (2380 kB/s)
-debconf: delaying package configuration, since apt-utils is not installed
-[...]
-Unpacking r-base-core (3.1.1-1+b2) ...
-Selecting previously unselected package r-base-dev.
-Preparing to unpack .../r-base-dev_3.1.1-1_all.deb ...
-Unpacking r-base-dev (3.1.1-1) ...
-Selecting previously unselected package r-cran-boot.
-Preparing to unpack .../r-cran-boot_1.3-13-1_all.deb ...
-Unpacking r-cran-boot (1.3-13-1) ...
-Selecting previously unselected package r-cran-mass.
-[...]
-Setting up r-base-core (3.1.1-1+b2) ...
-
-Creating config file /etc/R/Renviron with new version
-Setting up r-base-dev (3.1.1-1) ...
-Setting up r-cran-boot (1.3-13-1) ...
-Setting up r-cran-mass (7.3-34-1) ...
-Setting up r-cran-class (7.3-11-1) ...
+The following additional packages will be installed:
[...]
+done.
</code></pre>
</notextile>
<notextile>
<pre><code>root@fbf1d0f529d5:/# <span class="userinput">R</span>
-R version 3.1.1 (2014-07-10) -- "Sock it to Me"
-Copyright (C) 2014 The R Foundation for Statistical Computing
+R version 3.3.3 (2017-03-06) -- "Another Canoe"
+Copyright (C) 2017 The R Foundation for Statistical Computing
Platform: x86_64-pc-linux-gnu (64-bit)
R is free software and comes with ABSOLUTELY NO WARRANTY.
z1234:
API:
MaxItemsPerResponse: 1234
+ PostgreSQL:
+ Connection:
+ sslmode: require
+ Services:
+ RailsAPI:
+ InternalURLs:
+ "http://0.0.0.0:8000": {}
+ Workbench:
+ UserProfileFormFields:
+ color:
+ Type: select
+ Options:
+ fuchsia: {}
+ ApplicationMimetypesWithViewIcon:
+ whitespace: {}
`
code := CheckCommand.RunCommand("arvados config-check", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
c.Check(code, check.Equals, 0)
ManagementToken: ""
Services:
+
+ # In each of the service sections below, the keys under
+ # InternalURLs are the endpoints where the service should be
+ # listening, and reachable from other hosts in the cluster.
+ SAMPLE:
+ InternalURLs:
+ "http://example.host:12345": {}
+ SAMPLE: {}
+ ExternalURL: "-"
+
RailsAPI:
InternalURLs: {}
ExternalURL: "-"
user: ""
password: ""
dbname: ""
+ SAMPLE: ""
API:
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Set "Required" as "true" for any of these fields to make them required.
# If any of the required fields are missing in the user's profile, the user will be
# redirected to the profile page before they can access any Workbench features.
- UserProfileFormFields: {}
+ UserProfileFormFields:
+ SAMPLE:
+ Type: select
+ FormFieldTitle: Best color
+ FormFieldDescription: your favorite color
+ Required: false
+ Position: 1
+ Options:
+ red: {}
+ blue: {}
+ green: {}
+ SAMPLE: {}
+
# exampleTextValue: # key that will be set in properties
# Type: text #
# FormFieldTitle: ""
vnd.realvnc.bed: {}
xml: {}
xsl: {}
+ SAMPLE: {}
# The maximum number of bytes to load in the log viewer
LogViewerMaxBytes: 1M
ManagementToken: ""
Services:
+
+ # In each of the service sections below, the keys under
+ # InternalURLs are the endpoints where the service should be
+ # listening, and reachable from other hosts in the cluster.
+ SAMPLE:
+ InternalURLs:
+ "http://example.host:12345": {}
+ SAMPLE: {}
+ ExternalURL: "-"
+
RailsAPI:
InternalURLs: {}
ExternalURL: "-"
user: ""
password: ""
dbname: ""
+ SAMPLE: ""
API:
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Set "Required" as "true" for any of these fields to make them required.
# If any of the required fields are missing in the user's profile, the user will be
# redirected to the profile page before they can access any Workbench features.
- UserProfileFormFields: {}
+ UserProfileFormFields:
+ SAMPLE:
+ Type: select
+ FormFieldTitle: Best color
+ FormFieldDescription: your favorite color
+ Required: false
+ Position: 1
+ Options:
+ red: {}
+ blue: {}
+ green: {}
+ SAMPLE: {}
+
# exampleTextValue: # key that will be set in properties
# Type: text #
# FormFieldTitle: ""
vnd.realvnc.bed: {}
xml: {}
xsl: {}
+ SAMPLE: {}
# The maximum number of bytes to load in the log viewer
LogViewerMaxBytes: 1M
allowed[strings.ToLower(k)] = v
}
for k, vsupp := range supplied {
+ if k == "SAMPLE" {
+ // entry will be dropped in removeSampleKeys anyway
+ continue
+ }
vexp, ok := allowed[strings.ToLower(k)]
- if !ok && expected["SAMPLE"] != nil {
+ if expected["SAMPLE"] != nil {
vexp = expected["SAMPLE"]
} else if !ok {
log.Warnf("deprecated or unknown config entry: %s%s", prefix, k)
"io"
"os"
"os/exec"
+ "reflect"
"strings"
"testing"
c.Check(logs, check.HasLen, 2)
}
+func (s *LoadSuite) checkSAMPLEKeys(c *check.C, path string, x interface{}) {
+ v := reflect.Indirect(reflect.ValueOf(x))
+ switch v.Kind() {
+ case reflect.Map:
+ var stringKeys, sampleKey bool
+ iter := v.MapRange()
+ for iter.Next() {
+ k := iter.Key()
+ if k.Kind() == reflect.String {
+ stringKeys = true
+ if k.String() == "SAMPLE" || k.String() == "xxxxx" {
+ sampleKey = true
+ s.checkSAMPLEKeys(c, path+"."+k.String(), iter.Value().Interface())
+ }
+ }
+ }
+ if stringKeys && !sampleKey {
+ c.Errorf("%s is a map with string keys (type %T) but config.default.yml has no SAMPLE key", path, x)
+ }
+ return
+ case reflect.Struct:
+ for i := 0; i < v.NumField(); i++ {
+ val := v.Field(i)
+ if val.CanInterface() {
+ s.checkSAMPLEKeys(c, path+"."+v.Type().Field(i).Name, val.Interface())
+ }
+ }
+ }
+}
+
+func (s *LoadSuite) TestDefaultConfigHasAllSAMPLEKeys(c *check.C) {
+ cfg, err := Load(bytes.NewBuffer(DefaultYAML), ctxlog.TestLogger(c))
+ c.Assert(err, check.IsNil)
+ s.checkSAMPLEKeys(c, "", cfg)
+}
+
func (s *LoadSuite) TestNoUnrecognizedKeysInDefaultConfig(c *check.C) {
var logbuf bytes.Buffer
logger := logrus.New()
req := httptest.NewRequest("GET", "/arvados/v1/workflows/"+arvadostest.WorkflowWithDefinitionYAMLUUID, nil)
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- s.checkJSONErrorMatches(c, resp, `Not logged in`)
+ s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
}
func (s *FederationSuite) TestBadAuth(c *check.C) {
req.Header.Set("Authorization", "Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
- s.checkJSONErrorMatches(c, resp, `Not logged in`)
+ s.checkJSONErrorMatches(c, resp, `Not logged in.*`)
}
func (s *FederationSuite) TestNoAccess(c *check.C) {
req.Header.Set("Authorization", "Bearer "+arvadostest.SpectatorToken)
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
- s.checkJSONErrorMatches(c, resp, `.*not found`)
+ s.checkJSONErrorMatches(c, resp, `.*not found.*`)
}
func (s *FederationSuite) TestGetUnknownRemote(c *check.C) {
defer wg.Done()
err := cq.Unlock(uuid)
c.Check(err, check.NotNil)
- c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued*.")
+ c.Check(err, check.ErrorMatches, ".*cannot unlock when Queued.*")
err = cq.Lock(uuid)
c.Check(err, check.IsNil)
UnloggedAttributes []string
}
Collections struct {
- BlobSigning bool
- BlobSigningKey string
- BlobSigningTTL Duration
- CollectionVersioning bool
- DefaultTrashLifetime Duration
- DefaultReplication int
- ManagedProperties map[string]interface{}
+ BlobSigning bool
+ BlobSigningKey string
+ BlobSigningTTL Duration
+ CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]struct {
+ Value interface{}
+ Function string
+ Protected bool
+ }
PreserveVersionIfIdle Duration
TrashSweepInterval Duration
TrustAllContent bool
err = {}
end
err[:errors] ||= args
+ err[:errors].map! do |err|
+ err += " (" + Thread.current[:request_id] + ")"
+ end
err[:error_token] = [Time.now.utc.to_i, "%08x" % rand(16 ** 8)].join("+")
status = err.delete(:status) || 422
logger.error "Error #{err[:error_token]}: #{status}"
case "$TARGET" in
centos*)
- fpm_depends+=(libcurl-devel postgresql-devel)
+ fpm_depends+=(libcurl-devel postgresql-devel arvados-server)
;;
debian* | ubuntu*)
- fpm_depends+=(libcurl-ssl-dev libpq-dev g++)
+ fpm_depends+=(libcurl-ssl-dev libpq-dev g++ arvados-server)
;;
esac
token_time = token.split('+', 2).first.to_i
assert_operator(token_time, :>=, @start_stamp, "error token too old")
assert_operator(token_time, :<=, now_timestamp, "error token too new")
+ json_response['errors'].each do |err|
+ assert_match(/req-[a-z0-9]{20}/, err, "X-Request-Id value missing on error message")
+ end
end
def check_404(errmsg="Path not found")
assert_response 404
- assert_equal([errmsg], json_response['errors'])
+ json_response['errors'].each do |err|
+ assert(err.include?(errmsg), "error message '#{err}' expected to include '#{errmsg}'")
+ end
check_error_token
end
params: {specimen: {}},
headers: {'HTTP_ACCEPT' => ''})
assert_response 401
- assert_includes(json_response['errors'], "Not logged in")
+ json_response['errors'].each do |err|
+ assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+ end
end
test "login prompt respects JSON Accept header" do
params: {specimen: {}},
headers: {'HTTP_ACCEPT' => 'application/json'})
assert_response 401
- assert_includes(json_response['errors'], "Not logged in")
+ json_response['errors'].each do |err|
+ assert(err.include?("Not logged in"), "error message '#{err}' expected to include 'Not logged in'")
+ end
end
test "login prompt respects HTML Accept header" do
echo $vm_uuid > /var/lib/arvados/vm-uuid
fi
+if ! test -f /var/lib/arvados/api_database_pw ; then
+ ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
+fi
+database_pw=$(cat /var/lib/arvados/api_database_pw)
+
+if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
+ psql postgres -c "create user arvados with password '$database_pw'"
+fi
+psql postgres -c "ALTER USER arvados WITH SUPERUSER;"
+
+if test -a /usr/src/arvados/services/api/config/arvados_config.rb ; then
+ rm -f config/application.yml config/database.yml
+else
cat >config/application.yml <<EOF
$RAILS_ENV:
uuid_prefix: $uuid_prefix
EOF
(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
-
-if ! test -f /var/lib/arvados/api_database_pw ; then
- ruby -e 'puts rand(2**128).to_s(36)' > /var/lib/arvados/api_database_pw
-fi
-database_pw=$(cat /var/lib/arvados/api_database_pw)
-
-if ! (psql postgres -c "\du" | grep "^ arvados ") >/dev/null ; then
- psql postgres -c "create user arvados with password '$database_pw'"
-fi
-psql postgres -c "ALTER USER arvados WITH SUPERUSER;"
-
sed "s/password:.*/password: $database_pw/" <config/database.yml.example >config/database.yml
+fi
if ! test -f /var/lib/arvados/api_database_setup ; then
bundle exec rake db:setup
localip=$(ip addr show $defaultdev | grep 'inet ' | sed 's/ *inet \(.*\)\/.*/\1/')
fi
+root_cert=/var/lib/arvados/root-cert.pem
+root_cert_key=/var/lib/arvados/root-cert.key
+server_cert=/var/lib/arvados/server-cert-${localip}.pem
+server_cert_key=/var/lib/arvados/server-cert-${localip}.key
+
declare -A services
services=(
[workbench]=443
uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
-if test ! -s /var/lib/arvados/root-cert.pem ; then
+if ! openssl verify -CAfile $root_cert $root_cert ; then
# req signing request sub-command
# -new new certificate request
# -nodes "no des" don't encrypt key
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nbasicConstraints=critical,CA:true,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign")) \
- -out /var/lib/arvados/root-cert.pem \
- -keyout /var/lib/arvados/root-cert.key \
+ -out $root_cert \
+ -keyout $root_cert_key \
-days 365
- chown arvbox:arvbox /var/lib/arvados/root-cert.*
+ chown arvbox:arvbox $root_cert $root_cert_key
+ rm -f $server_cert $server_cert_key
fi
-if test ! -s /var/lib/arvados/server-cert-${localip}.pem ; then
+cp $root_cert /usr/local/share/ca-certificates/arvados-testing-cert.crt
+update-ca-certificates
+
+if ! openssl verify -CAfile $root_cert $server_cert ; then
+
+ rm -f $server_cert $server_cert_key
if [[ $localip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
san=IP:$localip
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
-out /var/lib/arvados/server-cert-${localip}.csr \
- -keyout /var/lib/arvados/server-cert-${localip}.key \
+ -keyout $server_cert_key \
-days 365
openssl x509 \
-req \
-in /var/lib/arvados/server-cert-${localip}.csr \
- -CA /var/lib/arvados/root-cert.pem \
- -CAkey /var/lib/arvados/root-cert.key \
- -out /var/lib/arvados/server-cert-${localip}.pem \
+ -CA $root_cert \
+ -CAkey $root_cert_key \
+ -out $server_cert \
-set_serial $RANDOM$RANDOM \
-extfile <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
-extensions x509_ext \
-days 365
- chown arvbox:arvbox /var/lib/arvados/server-cert-${localip}.*
+ chown arvbox:arvbox $server_cert $server_cert_key
fi
-cp /var/lib/arvados/root-cert.pem /usr/local/share/ca-certificates/arvados-testing-cert.crt
-update-ca-certificates
-
sv stop certificate
vm_uuid=$(cat /var/lib/arvados/vm-uuid)
database_pw=$(cat /var/lib/arvados/api_database_pw)
+workbench_secret_key_base=$(cat /var/lib/arvados/workbench_secret_token)
+
if test -s /var/lib/arvados/api_rails_env ; then
database_env=$(cat /var/lib/arvados/api_rails_env)
else
ExternalURL: "http://$localip:${services[arv-git-httpd]}/"
WebDAV:
ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
+ Composer:
+ ExternalURL: "http://$localip:${services[composer]}"
+ Controller:
+ ExternalURL: "https://$localip:${services[controller-ssl]}"
NodeProfiles: # to be deprecated in favor of "Services" section
"*":
arvados-controller:
AutoSetupNewUsers: true
AutoSetupNewUsersWithVmUUID: $vm_uuid
AutoSetupNewUsersWithRepository: true
+ Workbench:
+ SecretKeyBase: $workbench_secret_key_base
+ ArvadosDocsite: http://$localip:${services[doc]}/
EOF
/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
. /usr/local/lib/arvbox/common.sh
+openssl verify -CAfile $root_cert $server_cert
+
cat <<EOF >/var/lib/arvados/nginx.conf
worker_processes auto;
pid /var/lib/arvados/nginx.pid;
server {
listen *:${services[controller-ssl]} ssl default_server;
server_name controller;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://controller;
proxy_set_header Host \$http_host;
proxy_read_timeout 300s;
ssl on;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://arvados-ws;
server {
listen *:${services[workbench2-ssl]} ssl default_server;
server_name workbench2;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://workbench2;
proxy_set_header Host \$http_host;
server {
listen *:${services[keep-web-ssl]} ssl default_server;
server_name keep-web;
- ssl_certificate "/var/lib/arvados/server-cert-${localip}.pem";
- ssl_certificate_key "/var/lib/arvados/server-cert-${localip}.key";
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
location / {
proxy_pass http://keep-web;
proxy_set_header Host \$http_host;
fi
secret_token=$(cat /var/lib/arvados/sso_secret_token)
-test -s /var/lib/arvados/server-cert-${localip}.pem
+openssl verify -CAfile $root_cert $server_cert
cat >config/application.yml <<EOF
$RAILS_ENV:
fi
if test "$1" != "--only-deps" ; then
+ openssl verify -CAfile $root_cert $server_cert
exec bundle exec passenger start --port=${services[workbench]} \
--ssl --ssl-certificate=/var/lib/arvados/server-cert-${localip}.pem \
--ssl-certificate-key=/var/lib/arvados/server-cert-${localip}.key \
bundle exec passenger-config build-native-support
bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
-RAILS_GROUPS=assets bundle exec rake npm:install
if test "$1" = "--only-deps" ; then
- exit
+ # Workaround for validation that asserts there's a download URL
+ # configured, which breaks rake if it is missing.
+cat >config/application.yml <<EOF
+$RAILS_ENV:
+ keep_web_url: https://example.com/c=%{uuid_or_pdh}
+EOF
+ RAILS_GROUPS=assets bundle exec rake npm:install
+ rm config/application.yml
+ exit
fi
set -u
fi
secret_token=$(cat /var/lib/arvados/workbench_secret_token)
+if test -a /usr/src/arvados/apps/workbench/config/arvados_config.rb ; then
+ rm -f config/application.yml
+else
cat >config/application.yml <<EOF
$RAILS_ENV:
secret_token: $secret_token
workbench2_url: https://$localip:${services[workbench2-ssl]}
EOF
-bundle exec rake assets:precompile
-
(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
+fi
+
+RAILS_GROUPS=assets bundle exec rake npm:install
+bundle exec rake assets:precompile
def headHTML(self):
return '<link rel="stylesheet" href="{}">\n'.format(self.CSS)
- def chartdata(self, label, tasks, stat):
+ def chartdata(self, label, tasks, stats):
+ '''For Crunch2, label is the name of container request,
+ tasks is the top level container and
+ stats is index by a tuple of (category, metric).
+ '''
return {
- 'data': self._collate_data(tasks, stat),
+ 'data': self._collate_data(tasks, stats),
'options': {
+ 'legend': 'always',
'connectSeparatedPoints': True,
- 'labels': ['elapsed']+[uuid for uuid, _ in tasks.items()],
- 'title': '{}: {} {}'.format(label, stat[0], stat[1]),
+ 'labels': ['elapsed'] + stats[1],
+ 'title': '{}: {}'.format(label, stats[0]),
},
}
- def _collate_data(self, tasks, stat):
+ def _collate_data(self, tasks, stats):
data = []
nulls = []
+ # uuid is category for crunch2
for uuid, task in tasks.items():
- for pt in task.series[stat]:
- data.append([pt[0].total_seconds()] + nulls + [pt[1]])
+ # All stats in a category are assumed to have the same time base and same number of samples
+ category = stats[0]
+ series_names = stats[1]
+ sn0 = series_names[0]
+ series = task.series[(category,sn0)]
+ for i in range(len(series)):
+ pt = series[i]
+ vals = [task.series[(category,stat)][i][1] for stat in series_names[1:]]
+ data.append([pt[0].total_seconds()] + nulls + [pt[1]] + vals)
nulls.append(None)
return sorted(data)
try:
self.label = m.group('job_uuid')
except IndexError:
- self.label = 'container'
- if m.group('category').endswith(':'):
+ self.label = 'label #1'
+ category = m.group('category')
+ if category.endswith(':'):
# "stderr crunchstat: notice: ..."
continue
- elif m.group('category') in ('error', 'caught'):
+ elif category in ('error', 'caught'):
continue
- elif m.group('category') in ('read', 'open', 'cgroup', 'CID', 'Running'):
+ elif category in ('read', 'open', 'cgroup', 'CID', 'Running'):
# "stderr crunchstat: read /proc/1234/net/dev: ..."
# (old logs are less careful with unprefixed error messages)
continue
if group == 'interval' and this_interval_s:
stat = stat + '__rate'
val = val / this_interval_s
- if stat in ['user+sys__rate', 'tx+rx__rate']:
+ if stat in ['user+sys__rate', 'user__rate', 'sys__rate', 'tx+rx__rate', 'rx__rate', 'tx__rate']:
task.series[category, stat].append(
(timestamp - self.starttime, val))
else:
- if stat in ['rss']:
+ if stat in ['rss','used','total']:
task.series[category, stat].append(
(timestamp - self.starttime, val))
self.task_stats[task_id][category][stat] = val
(float(self.job_tot['blkio:0:0']['read']) /
float(self.job_tot['net:keep0']['rx']))
if self.job_tot['net:keep0']['rx'] > 0 else 0,
- lambda x: x * 100.0)):
+ lambda x: x * 100.0),
+ ('Temp disk utilization {}%',
+ (float(self.job_tot['statfs']['used']) /
+ float(self.job_tot['statfs']['total']))
+ if self.job_tot['statfs']['total'] > 0 else 0,
+ lambda x: x * 100.0),
+ ):
format_string, val, transform = args
if val == float('-Inf'):
continue
return itertools.chain(
self._recommend_cpu(),
self._recommend_ram(),
- self._recommend_keep_cache())
+ self._recommend_keep_cache(),
+ self._recommend_temp_disk(),
+ )
def _recommend_cpu(self):
"""Recommend asking for 4 cores if max CPU usage was 333%"""
math.ceil(asked_cache * 2 / self._runtime_constraint_mem_unit()))
+ def _recommend_temp_disk(self):
+ """Recommend decreasing temp disk if utilization < 50%"""
+ total = float(self.job_tot['statfs']['total'])
+ utilization = (float(self.job_tot['statfs']['used']) / total) if total > 0 else 0.0
+
+ if utilization < 50.8 and total > 0:
+ yield (
+ '#!! {} max temp disk utilization was {:.0f}% of {:.0f} MiB -- '
+ 'consider reducing "tmpdirMin" and/or "outdirMin"'
+ ).format(
+ self.label,
+ utilization * 100.0,
+ total / MB)
+
+
def _format(self, val):
"""Return a string representation of a stat.
'label': s.long_label(),
'charts': [
self.chartdata(s.label, s.tasks, stat)
- for stat in (('cpu', 'user+sys__rate'),
- ('mem', 'rss'),
- ('net:eth0', 'tx+rx__rate'),
- ('net:keep0', 'tx+rx__rate'))],
+ for stat in (('cpu', ['user+sys__rate', 'user__rate', 'sys__rate']),
+ ('mem', ['rss']),
+ ('net:eth0', ['tx+rx__rate','rx__rate','tx__rate']),
+ ('net:keep0', ['tx+rx__rate','rx__rate','tx__rate']),
+ ('statfs', ['used', 'total']),
+ )
+ ],
}
for s in self.summarizers]
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
-#!! container max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+# Temp disk utilization 1.21%
+#!! label #1 max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+#!! label #1 max temp disk utilization was 1% of 383960 MiB -- consider reducing "tmpdirMin" and/or "outdirMin"
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 1.21%
#!! container max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+#!! container max temp disk utilization was 1% of 383960 MiB -- consider reducing "tmpdirMin" and/or "outdirMin"
# Max network speed in a single interval: 42.58MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-jq0ekny1xou3zoh max RSS was 334 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-zvb2ocfycpomrup max RSS was 1 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-v831jm2uq0g2g9x max RSS was 1 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
return UTF8Decode(gzip.open(self.arvmountlog))
mock_cr().open.side_effect = _open
args = crunchstat_summary.command.ArgumentParser().parse_args(
- ['--job', self.fake_request['uuid']])
+ ['--container', self.fake_request['uuid']])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
self.diff_known_report(self.reportfile, cmd)
projects / arvados.git / commitdiff