sdk/python/tests/fed-migrate/*.cwlex
doc/install/*.xlsx
sdk/cwl/tests/wf/hello.txt
-sdk/cwl/tests/wf/indir1/hello2.txt
\ No newline at end of file
+sdk/cwl/tests/wf/indir1/hello2.txt
+sdk/cwl/tests/chipseq/data/Genomes/*
\ No newline at end of file
remote: https://rubygems.org/
specs:
RedCloth (4.3.2)
- actioncable (5.2.6)
- actionpack (= 5.2.6)
+ actioncable (5.2.6.3)
+ actionpack (= 5.2.6.3)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
+ actionmailer (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.6)
- actionview (= 5.2.6)
- activesupport (= 5.2.6)
+ actionpack (5.2.6.3)
+ actionview (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.6)
- activesupport (= 5.2.6)
+ actionview (5.2.6.3)
+ activesupport (= 5.2.6.3)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.6)
- activesupport (= 5.2.6)
+ activejob (5.2.6.3)
+ activesupport (= 5.2.6.3)
globalid (>= 0.3.6)
- activemodel (5.2.6)
- activesupport (= 5.2.6)
- activerecord (5.2.6)
- activemodel (= 5.2.6)
- activesupport (= 5.2.6)
+ activemodel (5.2.6.3)
+ activesupport (= 5.2.6.3)
+ activerecord (5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
arel (>= 9.0)
- activestorage (5.2.6)
- actionpack (= 5.2.6)
- activerecord (= 5.2.6)
+ activestorage (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
marcel (~> 1.0.0)
- activesupport (5.2.6)
+ activesupport (5.2.6.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
multipart-post (>= 1.2, < 3)
ffi (1.10.0)
flamegraph (0.9.5)
- globalid (0.4.2)
- activesupport (>= 4.2.0)
+ globalid (1.0.0)
+ activesupport (>= 5.0)
googleauth (0.9.0)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.10.0)
+ loofah (2.14.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (1.0.1)
+ marcel (1.0.2)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
mime-types (3.2.2)
mime-types-data (~> 3.2015)
mime-types-data (3.2019.0331)
- mini_mime (1.1.0)
- mini_portile2 (2.6.1)
+ mini_mime (1.1.2)
+ mini_portile2 (2.8.0)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh (5.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
- nio4r (2.5.7)
- nokogiri (1.12.5)
- mini_portile2 (~> 2.6.1)
+ nio4r (2.5.8)
+ nokogiri (1.13.3)
+ mini_portile2 (~> 2.8.0)
racc (~> 1.4)
npm-rails (0.2.1)
rails (>= 3.2)
rack (>= 1.2.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.6)
- actioncable (= 5.2.6)
- actionmailer (= 5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
- activemodel (= 5.2.6)
- activerecord (= 5.2.6)
- activestorage (= 5.2.6)
- activesupport (= 5.2.6)
+ rails (5.2.6.3)
+ actioncable (= 5.2.6.3)
+ actionmailer (= 5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
+ activestorage (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
bundler (>= 1.3.0)
- railties (= 5.2.6)
+ railties (= 5.2.6.3)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.3.0)
+ rails-html-sanitizer (1.4.2)
loofah (~> 2.3)
rails-perftest (0.0.7)
- railties (5.2.6)
- actionpack (= 5.2.6)
- activesupport (= 5.2.6)
+ railties (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
- rake (13.0.3)
+ rake (13.0.6)
raphael-rails (2.1.2)
rb-fsevent (0.10.3)
rb-inotify (0.10.0)
activesupport (>= 4.0)
sprockets (>= 3.0.0)
sshkey (2.0.0)
- thor (1.1.0)
+ thor (1.2.1)
thread_safe (0.3.6)
tilt (2.0.9)
tzinfo (1.2.9)
uglifier (2.7.2)
execjs (>= 0.3.0)
json (>= 1.8.0)
- websocket-driver (0.7.4)
+ websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
xpath (2.1.0)
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> /etc/gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
arv-put --version >/dev/null
-/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+PYTHON=`ls /usr/share/python3*/dist/python3-arvados-python-client/bin/python3 |head -n1`
+
+$PYTHON << EOF
import arvados
print("Successfully imported arvados")
EOF
Build only a specific package (or ONLY_BUILD from environment)
--arch <arch>
Build a specific architecture (or ARCH from environment, defaults to native architecture)
+--force-build
+ Build even if the package exists upstream or if it has already been
+ built locally
+--force-test
+ Test even if there is no new untested package
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,rc,target:,only-build:,arch:,build-version: \
+ help,debug,upload,rc,target:,force-test,only-build:,force-build,arch:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --force-test)
+ FORCE_TEST=1
+ ;;
+ --force-build)
+ FORCE_BUILD=1
+ ;;
--only-build)
ONLY_BUILD="$2"; shift
;;
build_args+=(--only-build "$ONLY_BUILD")
fi
+if [[ -n "$FORCE_BUILD" ]]; then
+ build_args+=(--force-build)
+fi
+
+if [[ -n "$FORCE_TEST" ]]; then
+ build_args+=(--force-test)
+fi
+
if [[ -n "$ARCH" ]]; then
build_args+=(--arch "$ARCH")
fi
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.5 or newer is required.
+Ruby 2.6 or newer is required.
* "Option 1: Install from packages":#packages
* "Option 2: Install with RVM":#rvm
h3. Centos 7
-The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install Ruby 2.5 or later.
+The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5 or newer, which is sufficient for Arvados.
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+
+Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
<notextile>
<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby ruby-dev</span></code></pre>
<notextile>
<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
</span></code></pre></notextile>
-This command installs the latest Ruby 2.5.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
<pre><code><span class="userinput">mkdir -p ~/src
cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.8.tar.gz | tar xz
-cd ruby-2.5.8
+curl -f https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz | tar xz
+cd ruby-2.7.5
./configure --disable-install-rdoc
make
sudo make install
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2022-02-10)
+h2(#main). development main (as of 2022-03-08)
"previous: Upgrading to 2.3.0":#v2_3_0
+h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
+
+Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
+
+This means that they are now installed under @/usr/share/python3.8@ (before, the path was @/usr/share/python3@). If you rely on the @python3@ executable from the packages (e.g. to load a virtualenv), you may need to update the path to that executable.
+
+h3. Minimum supported Ruby version is now 2.6
+
+The minimum supported Ruby version is now 2.6. If you are running Arvados on Debian 10 or Ubuntu 18.04, you may need to switch to using RVM or upgrade your OS. See "Install Ruby and Bundler":../install/ruby.html for more information.
+
h3. Anonymous token changes
The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary. If the anonymous token in @config.yml@ is specified as a full V2 token, that will now generate a warning - it should be updated to list just the secret (i.e. the part after the last forward slash).
The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config@. The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
+h2. Exported vocabulary definition
+
+When configured, the Controller also exports the "metadata vocabulary definition":{{site.baseurl}}/admin/metadata-vocabulary.html in JSON format. This functionality is useful for clients like Workbench2 and the Python SDK to provide "identifier to human-readable labels" translations facilities for reading and writing objects on the system. This is available at @/arvados/v1/vocabulary@, for example @https://{{ site.arvados_api_host }}/arvados/v1/vocabulary@.
+
h2. Workbench examples
Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
h2(#supportedlinux). Supported GNU/Linux distributions
table(table table-bordered table-condensed).
-|_. Distribution|_. State|_. Last supported version|
+|_. Distribution|_. State|_. Last supported Arvados version|
|CentOS 7|Supported|Latest|
|Debian 11 ("bullseye")|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
## arvados controller
## arvados websocket
## arvados cloud dispatcher
+## arvados keepbalance
# WORKBENCH node:
## arvados workbench
## arvados workbench2
h4. API
<notextile>
<pre><code>scp -r provision.sh local* user@host:
-ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher
+ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher,keepbalance
</code></pre>
</notextile>
for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
print("got collection " + c["uuid"])
{% endcodeblock %}
+
+h2. Querying the vocabulary definition
+
+The Python SDK provides facilities to interact with the "active metadata vocabulary":{{ site.baseurl }}/admin/metadata-vocabulary.html in the system. The developer can do key and value lookups in a case-insensitive manner:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+[k.identifier for k in set(voc.key_aliases.values())]
+# Example output: ['IDTAGCOLORS', 'IDTAGFRUITS', 'IDTAGCOMMENT', 'IDTAGIMPORTANCES', 'IDTAGCATEGORIES', 'IDTAGSIZES', 'IDTAGANIMALS']
+voc['IDTAGSIZES'].preferred_label
+# Example output: 'Size'
+[v.preferred_label for v in set(voc['size'].value_aliases.values())]
+# Example output: ['S', 'M', 'L', 'XL', 'XS']
+voc['size']['s'].aliases
+# Example output: ['S', 'small']
+voc['size']['Small'].identifier
+# Example output: 'IDVALSIZES2'
+{% endcodeblock %}
+
+h2. Translating between vocabulary identifiers and labels
+
+Client software might need to present properties to the user in a human-readable form or take input from the user without requiring them to remember identifiers. For these cases, there're a couple of conversion methods that take a dictionary as input like this:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+voc.convert_to_labels({'IDTAGIMPORTANCES': 'IDVALIMPORTANCES1'})
+# Example output: {'Importance': 'Critical'}
+voc.convert_to_identifiers({'creature': 'elephant'})
+# Example output: {'IDTAGANIMALS': 'IDVALANIMALS3'}
+{% endcodeblock %}
\ No newline at end of file
If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
+
<notextile>
<pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
(python-arvados-python-client) ~$ <code class="userinput">python</code>
RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version
# use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python
-RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
+RUN PYTHON=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python|head -n1` && rm -f /usr/bin/python && ln -s $PYTHON /usr/bin/python
+RUN PYTHON3=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python3|head -n1` && rm -f /usr/bin/python3 && ln -s $PYTHON3 /usr/bin/python3
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
- github.com/containerd/containerd v1.5.8 // indirect
+ github.com/containerd/containerd v1.5.10 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/go-connections v0.3.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
- github.com/opencontainers/image-spec v1.0.1 // indirect
+ github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pelletier/go-buffruneio v0.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
+github.com/containerd/containerd v1.5.10 h1:3cQ2uRVCkJVcx5VombsE7105Gl9Wrl7ORAO3+4+ogf4=
+github.com/containerd/containerd v1.5.10/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
+github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
details.
- cudaComputeCapabilityMin:
- type: string
- doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
- deviceCountMin:
- type: int?
+ cudaComputeCapability:
+ type:
+ - 'string'
+ - 'string[]'
+ doc: |
+ CUDA hardware capability required to run the software, in X.Y
+ format.
+
+ * If this is a single value, it defines only the minimum
+ compute capability. GPUs with higher capability are also
+ accepted.
+
+ * If it is an array value, then only select GPUs with compute
+ capabilities that explicitly appear in the array.
+ cudaDeviceCountMin:
+ type: ['null', int, cwl:Expression]
default: 1
- doc: Minimum number of GPU devices to request, default 1.
- deviceCountMax:
- type: int?
- doc: Maximum number of GPU devices to request. If not specified, same as `deviceCountMin`.
+ doc: |
+ Minimum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMax`. If neither are specified,
+ default 1.
+ cudaDeviceCountMax:
+ type: ['null', int, cwl:Expression]
+ doc: |
+ Maximum number of GPU devices to request. If not specified,
+ same as `cudaDeviceCountMin`.
cuda_req, _ = self.get_requirement("http://commonwl.org/cwltool#CUDARequirement")
if cuda_req:
runtime_constraints["cuda"] = {
- "device_count": cuda_req.get("deviceCountMin", 1),
+ "device_count": resources.get("cudaDeviceCount", 1),
"driver_version": cuda_req["cudaVersionMin"],
- "hardware_capability": cuda_req["cudaComputeCapabilityMin"]
+ "hardware_capability": aslist(cuda_req["cudaComputeCapability"])[0]
}
if self.timelimit is not None and self.timelimit > 0:
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20220217222804',
+ 'cwltool==3.1.20220224085855',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
--- /dev/null
+{
+ "referenceGenomeSequence": {
+ "class": "File",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "referenceGenomeSequenceDrosophila": {
+ "class": "File",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "blacklistBed": {
+ "class": "File",
+ "location": "data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieHumanReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Homo sapiens",
+ "version": "hg38"
+ },
+ "annotation": {
+ "source": "gencode",
+ "version": "v24"
+ }
+ }
+ },
+ "BowtieDrosophilaReference": {
+ "class": "Directory",
+ "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/",
+ "metadata": {
+ "reference_genome": {
+ "organism": "Drosophila melanogaster",
+ "version": "rmel_r6.16"
+ }
+ }
+ },
+ "sampleName": "LED054_0p03nMR1.0",
+ "inputFastq1": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastq2": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz",
+ "secondaryFiles": []
+ },
+ "inputFastqUMI": {
+ "class": "File",
+ "metadata": {
+ "user": "kmavrommatis",
+ "sample_id": [
+ 2
+ ]
+ },
+ "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz",
+ "secondaryFiles": []
+ }
+}
+
--- /dev/null
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "id": "#main",
+ "doc": "Pipeline that is applied on single ChIP-seq samples.\n\nStarts with QC on the reads and trimming (for adapters and based on quality)\n\nAligns to human genome and adds UMI\n\nAligns to Drosophila genome and counts the number of reads.\n\nAfter the alignment to human genome the files are filtered for duplicates, multimappers and alignments in black listed regions",
+ "label": "ChIP-Seq (single sample)",
+ "inputs": [
+ {
+ "id": "#inputFastq1",
+ "type": "File",
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1726.25
+ },
+ {
+ "id": "#blacklistBed",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 746.4744873046875,
+ "https://www.sevenbridges.com/y": 1903.265625
+ },
+ {
+ "id": "#referenceGenomeSequence",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai",
+ "^.dict"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fasta, fa",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1405.203125
+ },
+ {
+ "id": "#sampleName",
+ "type": "string",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1191.171875
+ },
+ {
+ "id": "#inputFastq2",
+ "type": [
+ "null",
+ "File"
+ ],
+ "https://www.sevenbridges.com/fileTypes": "fastq",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1619.234375
+ },
+ {
+ "id": "#inputFastqUMI",
+ "type": "File",
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1512.21875
+ },
+ {
+ "id": "#BowtieHumanReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1519.21875
+ },
+ {
+ "id": "#BowtieDrosophilaReference",
+ "type": "Directory",
+ "https://www.sevenbridges.com/x": 363.875,
+ "https://www.sevenbridges.com/y": 1626.234375
+ },
+ {
+ "id": "#referenceGenomeSequenceDrosophila",
+ "type": "File",
+ "secondaryFiles": [
+ ".fai"
+ ],
+ "https://www.sevenbridges.com/x": 0,
+ "https://www.sevenbridges.com/y": 1298.1875
+ }
+ ],
+ "outputs": [
+ ],
+ "steps": [
+ {
+ "id": "#step1",
+ "in": {
+ "inp": "#inputFastq1"
+ },
+ "out": [],
+ "run": "../cat.cwl"
+ }
+ ],
+ "requirements": [
+ ]
+ },
+ ],
+ "cwlVersion": "v1.0"
+}
import unittest
import os
import functools
+import threading
import cwltool.process
import cwltool.secrets
import cwltool.load_tool
"basedir": "",
"make_fs_access": make_fs_access,
"construct_tool_object": runner.arv_make_tool,
- "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
+ "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
+ "loader": Loader({}),
+ "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"name": "test_run_"+str(enable_reuse),
"make_fs_access": make_fs_access,
"tmpdir": "/tmp",
+ "outdir": "/tmp",
"enable_reuse": enable_reuse,
"priority": 500,
- "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "workflow_eval_lock": threading.Condition(threading.RLock())
})
if isinstance(runner, mock.MagicMock):
runner.api.collections().get().execute.return_value = {
"portable_data_hash": "99999999999999999999999999999993+99"}
- tool = cmap({
- "inputs": [],
- "outputs": [],
- "baseCommand": "nvidia-smi",
- "arguments": [],
- "id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool",
- "requirements": [
- {
+ test_cwl_req = [{
"class": "http://commonwl.org/cwltool#CUDARequirement",
"cudaVersionMin": "11.0",
- "cudaComputeCapabilityMin": "9.0",
- }
- ]
- })
+ "cudaComputeCapability": "9.0",
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": "9.0",
+ "cudaDeviceCountMin": 2
+ }, {
+ "class": "http://commonwl.org/cwltool#CUDARequirement",
+ "cudaVersionMin": "11.0",
+ "cudaComputeCapability": ["4.0", "5.0"],
+ "cudaDeviceCountMin": 2
+ }]
+
+ test_arv_req = [{
+ 'device_count': 1,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "9.0"
+ }, {
+ 'device_count': 2,
+ 'driver_version': "11.0",
+ 'hardware_capability': "4.0"
+ }]
+
+ for test_case in range(0, len(test_cwl_req)):
- loadingContext, runtimeContext = self.helper(runner, True)
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "nvidia-smi",
+ "arguments": [],
+ "id": "",
+ "cwlVersion": "v1.2",
+ "class": "CommandLineTool",
+ "requirements": [test_cwl_req[test_case]]
+ })
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
- arvtool.formatgraph = None
+ loadingContext, runtimeContext = self.helper(runner, True)
- for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
- j.run(runtimeContext)
- runner.api.container_requests().create.assert_called_with(
- body=JsonDiffMatcher({
- 'environment': {
- 'HOME': '/var/spool/cwl',
- 'TMPDIR': '/tmp'
- },
- 'name': 'test_run_True',
- 'runtime_constraints': {
- 'vcpus': 1,
- 'ram': 268435456,
- 'cuda': {
- 'device_count': 1,
- 'driver_version': "11.0",
- 'hardware_capability': "9.0"
- }
- },
- 'use_existing': True,
- 'priority': 500,
- 'mounts': {
- '/tmp': {'kind': 'tmp',
- "capacity": 1073741824
- },
- '/var/spool/cwl': {'kind': 'tmp',
- "capacity": 1073741824 }
- },
- 'state': 'Committed',
- 'output_name': 'Output for step test_run_True',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'output_path': '/var/spool/cwl',
- 'output_ttl': 0,
- 'container_image': '99999999999999999999999999999993+99',
- 'command': ['nvidia-smi'],
- 'cwd': '/var/spool/cwl',
- 'scheduling_parameters': {},
- 'properties': {},
- 'secret_mounts': {},
- 'output_storage_classes': ["default"]
- }))
+ arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+ runner.api.container_requests().create.assert_called_with(
+ body=JsonDiffMatcher({
+ 'environment': {
+ 'HOME': '/var/spool/cwl',
+ 'TMPDIR': '/tmp'
+ },
+ 'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'runtime_constraints': {
+ 'vcpus': 1,
+ 'ram': 268435456,
+ 'cuda': test_arv_req[test_case]
+ },
+ 'use_existing': True,
+ 'priority': 500,
+ 'mounts': {
+ '/tmp': {'kind': 'tmp',
+ "capacity": 1073741824
+ },
+ '/var/spool/cwl': {'kind': 'tmp',
+ "capacity": 1073741824 }
+ },
+ 'state': 'Committed',
+ 'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'output_path': '/var/spool/cwl',
+ 'output_ttl': 0,
+ 'container_image': '99999999999999999999999999999993+99',
+ 'command': ['nvidia-smi'],
+ 'cwd': '/var/spool/cwl',
+ 'scheduling_parameters': {},
+ 'properties': {},
+ 'secret_mounts': {},
+ 'output_storage_classes': ["default"]
+ }))
# The test passes no builder.resources
"baseCommand": "echo",
"arguments": [],
"id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool"
+ "cwlVersion": "v1.0",
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner, True)
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
container_request = {
'name': 'test_run_True',
'runtime_constraints': {
'vcpus': 1,
- 'ram': 268435456
+ 'ram': 1073741824,
},
'use_existing': True,
'priority': 500,
svc.insecure = insecure
svc.request_id = request_id
svc.config = lambda: util.get_config_once(svc)
+ svc.vocabulary = lambda: util.get_vocabulary_once(svc)
kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
kwargs['http'].cache = None
kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
return (None, None)
fn = os.path.join(root, ".arvados#collection")
if os.path.exists(fn):
- with file(fn, 'r') as f:
+ with open(fn, 'r') as f:
c = json.load(f)
return (c["portable_data_hash"], branch)
else:
if not hasattr(svc, '_cached_config'):
svc._cached_config = svc.configs().get().execute()
return svc._cached_config
+
+def get_vocabulary_once(svc):
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+from . import api
+
+_logger = logging.getLogger('arvados.vocabulary')
+
+def load_vocabulary(api_client=None):
+ """Load the Arvados vocabulary from the API.
+ """
+ if api_client is None:
+ api_client = api('v1')
+ return Vocabulary(api_client.vocabulary())
+
+class VocabularyError(Exception):
+ """Base class for all vocabulary errors.
+ """
+ pass
+
+class VocabularyKeyError(VocabularyError):
+ pass
+
+class VocabularyValueError(VocabularyError):
+ pass
+
+class Vocabulary(object):
+ def __init__(self, voc_definition={}):
+ self.strict_keys = voc_definition.get('strict_tags', False)
+ self.key_aliases = {}
+
+ for key_id, val in (voc_definition.get('tags') or {}).items():
+ strict = val.get('strict', False)
+ key_labels = [l['label'] for l in val.get('labels', [])]
+ values = {}
+ for v_id, v_val in (val.get('values') or {}).items():
+ labels = [l['label'] for l in v_val.get('labels', [])]
+ values[v_id] = VocabularyValue(v_id, labels)
+ vk = VocabularyKey(key_id, key_labels, values, strict)
+ self.key_aliases[key_id.lower()] = vk
+ for alias in vk.aliases:
+ self.key_aliases[alias.lower()] = vk
+
+ def __getitem__(self, key):
+ return self.key_aliases[key.lower()]
+
+ def convert_to_identifiers(self, obj={}):
+ """Translate key/value pairs to machine readable identifiers.
+ """
+ return self._convert_to_what(obj, 'identifier')
+
+ def convert_to_labels(self, obj={}):
+ """Translate key/value pairs to human readable labels.
+ """
+ return self._convert_to_what(obj, 'preferred_label')
+
+ def _convert_to_what(self, obj={}, what=None):
+ if not isinstance(obj, dict):
+ raise ValueError("obj must be a dict")
+ if what not in ['preferred_label', 'identifier']:
+ raise ValueError("what attr must be 'preferred_label' or 'identifier'")
+ r = {}
+ for k, v in obj.items():
+ # Key validation & lookup
+ key_found = False
+ if not isinstance(k, str):
+ raise VocabularyKeyError("key '{}' must be a string".format(k))
+ k_what, v_what = k, v
+ try:
+ k_what = getattr(self[k], what)
+ key_found = True
+ except KeyError:
+ if self.strict_keys:
+ raise VocabularyKeyError("key '{}' not found in vocabulary".format(k))
+
+ # Value validation & lookup
+ if isinstance(v, list):
+ v_what = []
+ for x in v:
+ if not isinstance(x, str):
+ raise VocabularyValueError("value '{}' for key '{}' must be a string".format(x, k))
+ try:
+ v_what.append(getattr(self[k][x], what))
+ except KeyError:
+ if self[k].strict:
+ raise VocabularyValueError("value '{}' not found for key '{}'".format(x, k))
+ v_what.append(x)
+ else:
+ if not isinstance(v, str):
+ raise VocabularyValueError("{} value '{}' for key '{}' must be a string".format(type(v).__name__, v, k))
+ try:
+ v_what = getattr(self[k][v], what)
+ except KeyError:
+ if key_found and self[k].strict:
+ raise VocabularyValueError("value '{}' not found for key '{}'".format(v, k))
+
+ r[k_what] = v_what
+ return r
+
+class VocabularyData(object):
+ def __init__(self, identifier, aliases=[]):
+ self.identifier = identifier
+ self.aliases = aliases
+
+ def __getattribute__(self, name):
+ if name == 'preferred_label':
+ return self.aliases[0]
+ return super(VocabularyData, self).__getattribute__(name)
+
+class VocabularyValue(VocabularyData):
+ def __init__(self, identifier, aliases=[]):
+ super(VocabularyValue, self).__init__(identifier, aliases)
+
+class VocabularyKey(VocabularyData):
+ def __init__(self, identifier, aliases=[], values={}, strict=False):
+ super(VocabularyKey, self).__init__(identifier, aliases)
+ self.strict = strict
+ self.value_aliases = {}
+ for v_id, v_val in values.items():
+ self.value_aliases[v_id.lower()] = v_val
+ for v_alias in v_val.aliases:
+ self.value_aliases[v_alias.lower()] = v_val
+
+ def __getitem__(self, key):
+ return self.value_aliases[key.lower()]
\ No newline at end of file
'google-api-python-client >=1.6.2, <2',
'google-auth<2',
'httplib2 >=0.9.2, <0.20.2',
- 'pycurl >=7.19.5.1',
+ 'pycurl >=7.19.5.1, <7.45.0',
'ruamel.yaml >=0.15.54, <0.17.11',
'setuptools',
'ws4py >=0.4.2',
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import unittest
+import mock
+
+from arvados import api, vocabulary
+
+class VocabularyTest(unittest.TestCase):
+ EXAMPLE_VOC = {
+ 'tags': {
+ 'IDTAGANIMALS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Animal'},
+ {'label': 'Creature'},
+ ],
+ 'values': {
+ 'IDVALANIMAL1': {
+ 'labels': [
+ {'label': 'Human'},
+ {'label': 'Homo sapiens'},
+ ],
+ },
+ 'IDVALANIMAL2': {
+ 'labels': [
+ {'label': 'Elephant'},
+ {'label': 'Loxodonta'},
+ ],
+ },
+ },
+ },
+ 'IDTAGIMPORTANCES': {
+ 'strict': True,
+ 'labels': [
+ {'label': 'Importance'},
+ {'label': 'Priority'},
+ ],
+ 'values': {
+ 'IDVALIMPORTANCE1': {
+ 'labels': [
+ {'label': 'High'},
+ {'label': 'High priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE2': {
+ 'labels': [
+ {'label': 'Medium'},
+ {'label': 'Medium priority'},
+ ],
+ },
+ 'IDVALIMPORTANCE3': {
+ 'labels': [
+ {'label': 'Low'},
+ {'label': 'Low priority'},
+ ],
+ },
+ },
+ },
+ 'IDTAGCOMMENTS': {
+ 'strict': False,
+ 'labels': [
+ {'label': 'Comment'},
+ {'label': 'Notes'},
+ ],
+ 'values': None,
+ },
+ },
+ }
+
+ def setUp(self):
+ self.api = arvados.api('v1')
+ self.voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ self.api.vocabulary = mock.MagicMock(return_value=self.EXAMPLE_VOC)
+
+ def test_vocabulary_keys(self):
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(
+ self.voc.key_aliases.keys(),
+ set(['idtaganimals', 'creature', 'animal',
+ 'idtagimportances', 'importance', 'priority',
+ 'idtagcomments', 'comment', 'notes'])
+ )
+
+ vk = self.voc.key_aliases['creature']
+ self.assertEqual(vk.strict, False)
+ self.assertEqual(vk.identifier, 'IDTAGANIMALS')
+ self.assertEqual(vk.aliases, ['Animal', 'Creature'])
+ self.assertEqual(vk.preferred_label, 'Animal')
+ self.assertEqual(
+ vk.value_aliases.keys(),
+ set(['idvalanimal1', 'human', 'homo sapiens',
+ 'idvalanimal2', 'elephant', 'loxodonta'])
+ )
+
+ def test_vocabulary_values(self):
+ vk = self.voc.key_aliases['creature']
+ vv = vk.value_aliases['human']
+ self.assertEqual(vv.identifier, 'IDVALANIMAL1')
+ self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
+ self.assertEqual(vv.preferred_label, 'Human')
+
+ def test_vocabulary_indexing(self):
+ self.assertEqual(self.voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(self.voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+ with self.assertRaises(KeyError):
+ inexistant = self.voc['foo']
+
+ def test_empty_vocabulary(self):
+ voc = vocabulary.Vocabulary({})
+ self.assertEqual(voc.strict_keys, False)
+ self.assertEqual(voc.key_aliases, {})
+
+ def test_load_vocabulary_with_api(self):
+ voc = vocabulary.load_vocabulary(self.api)
+ self.assertEqual(voc['creature']['human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+ self.assertEqual(voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+
+ def test_convert_to_identifiers(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_value_lists(self):
+ cases = [
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'IDTAGIMPORTANCES': ['High', 'Medium']},
+ {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'priority': ['high', 'medium']},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_identifiers(case),
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_identifiers_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ strict_voc.convert_to_identifiers({'foo': 'bar'})
+
+ def test_convert_to_identifiers_invalid_key(self):
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({42: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({None: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_identifiers({('f', 'o', 'o'): 'bar'})
+
+ def test_convert_to_identifiers_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_identifiers({'Animal': 'foo'}), {'IDTAGANIMALS': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Priority': 'foo'})
+
+ def test_convert_to_identifiers_invalid_value(self):
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': 42})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': None})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': {'hello': 'world'}})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [42]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [None]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Animal': [{'hello': 'world'}]})
+
+ def test_convert_to_identifiers_unknown_value_list(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(
+ self.voc.convert_to_identifiers({'Animal': ['foo', 'loxodonta']}),
+ {'IDTAGANIMALS': ['foo', 'IDVALANIMAL2']}
+ )
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_identifiers({'Priority': ['foo', 'bar']})
+
+ def test_convert_to_labels(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+ {'IDTAGIMPORTANCES': 'High'},
+ {'importance': 'IDVALIMPORTANCE1'},
+ {'priority': 'high priority'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_multiple_pairs(self):
+ cases = [
+ {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+ {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+ {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+ {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': 'High', 'Animal': 'Human', 'Comment': 'Very important person'},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_value_lists(self):
+ cases = [
+ {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'IDTAGIMPORTANCES': ['High', 'Medium']},
+ {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+ {'priority': ['high', 'medium']},
+ ]
+ for case in cases:
+ self.assertEqual(
+ self.voc.convert_to_labels(case),
+ {'Importance': ['High', 'Medium']},
+ "failing test case: {}".format(case)
+ )
+
+ def test_convert_to_labels_unknown_key(self):
+ # Non-strict vocabulary
+ self.assertEqual(self.voc.strict_keys, False)
+ self.assertEqual(self.voc.convert_to_labels({'foo': 'bar'}), {'foo': 'bar'})
+ # Strict vocabulary
+ strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+ strict_voc.strict_keys = True
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ strict_voc.convert_to_labels({'foo': 'bar'})
+
+ def test_convert_to_labels_invalid_key(self):
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({42: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({None: 'bar'})
+ with self.assertRaises(vocabulary.VocabularyKeyError):
+ self.voc.convert_to_labels({('f', 'o', 'o'): 'bar'})
+
+ def test_convert_to_labels_unknown_value(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(self.voc.convert_to_labels({'IDTAGANIMALS': 'foo'}), {'Animal': 'foo'})
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': 'foo'})
+
+ def test_convert_to_labels_invalid_value(self):
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': {'high': True}})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': None})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': 42})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': False})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [42]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [None]})
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': [{'high': True}]})
+
+ def test_convert_to_labels_unknown_value_list(self):
+ # Non-strict key
+ self.assertEqual(self.voc['animal'].strict, False)
+ self.assertEqual(
+ self.voc.convert_to_labels({'IDTAGANIMALS': ['foo', 'IDVALANIMAL1']}),
+ {'Animal': ['foo', 'Human']}
+ )
+ # Strict key
+ self.assertEqual(self.voc['priority'].strict, True)
+ with self.assertRaises(vocabulary.VocabularyValueError):
+ self.voc.convert_to_labels({'IDTAGIMPORTANCES': ['foo', 'bar']})
+
+ def test_convert_roundtrip(self):
+ initial = {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'}
+ converted = self.voc.convert_to_labels(initial)
+ self.assertNotEqual(converted, initial)
+ self.assertEqual(self.voc.convert_to_identifiers(converted), initial)
GEM
remote: https://rubygems.org/
specs:
- actioncable (5.2.6)
- actionpack (= 5.2.6)
+ actioncable (5.2.6.3)
+ actionpack (= 5.2.6.3)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
- actionmailer (5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
+ actionmailer (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.2.6)
- actionview (= 5.2.6)
- activesupport (= 5.2.6)
+ actionpack (5.2.6.3)
+ actionview (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.2.6)
- activesupport (= 5.2.6)
+ actionview (5.2.6.3)
+ activesupport (= 5.2.6.3)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.3)
- activejob (5.2.6)
- activesupport (= 5.2.6)
+ activejob (5.2.6.3)
+ activesupport (= 5.2.6.3)
globalid (>= 0.3.6)
- activemodel (5.2.6)
- activesupport (= 5.2.6)
- activerecord (5.2.6)
- activemodel (= 5.2.6)
- activesupport (= 5.2.6)
+ activemodel (5.2.6.3)
+ activesupport (= 5.2.6.3)
+ activerecord (5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
arel (>= 9.0)
- activestorage (5.2.6)
- actionpack (= 5.2.6)
- activerecord (= 5.2.6)
+ activestorage (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
marcel (~> 1.0.0)
- activesupport (5.2.6)
+ activesupport (5.2.6.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
faraday (0.15.4)
multipart-post (>= 1.2, < 3)
ffi (1.9.25)
- globalid (0.4.2)
- activesupport (>= 4.2.0)
+ globalid (1.0.0)
+ activesupport (>= 5.0)
googleauth (0.9.0)
faraday (~> 0.12)
jwt (>= 1.4, < 3.0)
railties (>= 4)
request_store (~> 1.0)
logstash-event (1.2.02)
- loofah (2.10.0)
+ loofah (2.14.0)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
mini_mime (>= 0.1.1)
- marcel (1.0.1)
+ marcel (1.0.2)
memoist (0.16.2)
metaclass (0.0.4)
method_source (1.0.0)
- mini_mime (1.1.0)
- mini_portile2 (2.6.1)
+ mini_mime (1.1.2)
+ mini_portile2 (2.8.0)
minitest (5.10.3)
mocha (1.8.0)
metaclass (~> 0.0.1)
net-ssh (5.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
- nio4r (2.5.7)
- nokogiri (1.12.5)
- mini_portile2 (~> 2.6.1)
+ nio4r (2.5.8)
+ nokogiri (1.13.3)
+ mini_portile2 (~> 2.8.0)
racc (~> 1.4)
oj (3.9.2)
optimist (3.0.0)
rack (2.2.3)
rack-test (1.1.0)
rack (>= 1.0, < 3)
- rails (5.2.6)
- actioncable (= 5.2.6)
- actionmailer (= 5.2.6)
- actionpack (= 5.2.6)
- actionview (= 5.2.6)
- activejob (= 5.2.6)
- activemodel (= 5.2.6)
- activerecord (= 5.2.6)
- activestorage (= 5.2.6)
- activesupport (= 5.2.6)
+ rails (5.2.6.3)
+ actioncable (= 5.2.6.3)
+ actionmailer (= 5.2.6.3)
+ actionpack (= 5.2.6.3)
+ actionview (= 5.2.6.3)
+ activejob (= 5.2.6.3)
+ activemodel (= 5.2.6.3)
+ activerecord (= 5.2.6.3)
+ activestorage (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
bundler (>= 1.3.0)
- railties (= 5.2.6)
+ railties (= 5.2.6.3)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.4)
actionpack (>= 5.0.1.x)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
- rails-html-sanitizer (1.3.0)
+ rails-html-sanitizer (1.4.2)
loofah (~> 2.3)
rails-observers (0.1.5)
activemodel (>= 4.0)
rails-perftest (0.0.7)
- railties (5.2.6)
- actionpack (= 5.2.6)
- activesupport (= 5.2.6)
+ railties (5.2.6.3)
+ actionpack (= 5.2.6.3)
+ activesupport (= 5.2.6.3)
method_source
rake (>= 0.8.7)
thor (>= 0.19.0, < 2.0)
- rake (13.0.3)
+ rake (13.0.6)
rb-fsevent (0.10.3)
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
sshkey (2.0.0)
test-unit (3.3.1)
power_assert
- thor (1.1.0)
+ thor (1.2.1)
thread_safe (0.3.6)
tzinfo (1.2.9)
thread_safe (~> 0.1)
- websocket-driver (0.7.4)
+ websocket-driver (0.7.5)
websocket-extensions (>= 0.1.0)
websocket-extensions (0.1.5)
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20210628",
+ revision: "20220222",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
}
}
+ discovery[:resources]['vocabularies'] = {
+ methods: {
+ get: {
+ id: "arvados.vocabularies.get",
+ path: "vocabulary",
+ httpMethod: "GET",
+ description: "Get vocabulary definition",
+ parameters: {
+ },
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
+ },
+ }
+ }
+
discovery[:resources]['sys'] = {
methods: {
get: {
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+ pysdk_dep = "=={}".format(version)
+else:
+ # On dev releases, arvados-python-client may have a different timestamp
+ pysdk_dep = "<={}".format(version)
short_tests_only = False
if '--short-tests-only' in sys.argv:
('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client',
+ 'arvados-python-client{}'.format(pysdk_dep),
],
test_suite='tests',
tests_require=['pbr<1.7.0', 'mock>=1.0'],
inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
cp -vr /vagrant/tests /home/vagrant/tests;
sed 's#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"zeppo.local\"#g;
+ 's#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g;
s#cluster_fixme_or_this_wont_work#zeppo#g;
s#domain_fixme_or_this_wont_work#local#g;' \
/vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
DispatchCloud:
InternalURLs:
'http://__CONTROLLER_INT_IP__:9006': {}
+ Keepbalance:
+ InternalURLs:
+ 'http://localhost:9005': {}
Keepproxy:
ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__'
InternalURLs:
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
'http://__CLUSTER__.__DOMAIN__:9006': {}
Keepbalance:
InternalURLs:
- 'http://__CLUSTER__.__DOMAIN__:9005': {}
+ 'http://localhost:9005': {}
Keepproxy:
ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__'
InternalURLs:
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
host: 127.0.0.1
password: "__DATABASE_PASSWORD__"
user: __CLUSTER___arvados
- encoding: en_US.utf8
- client_encoding: UTF8
+ extra_conn_params:
+ client_encoding: UTF8
+ # Centos7 does not enable SSL by default, so we disable
+ # it here just for testing of the formula purposes only.
+ # You should not do this in production, and should
+ # configure Postgres certificates correctly
+ {%- if grains.os_family in ('RedHat',) %}
+ sslmode: disable
+ {%- endif %}
tls:
# certificate: ''
# When using arvados-snakeoil certs set insecure: true
insecure: true
+ resources:
+ virtual_machines:
+ shell:
+ name: webshell
+ backend: 127.0.1.1
+ port: 4200
+
### TOKENS
tokens:
system_root: __SYSTEM_ROOT_TOKEN__
ExternalURL: 'https://__HOSTNAME_EXT__:__CONTROLLER_EXT_SSL_PORT__'
InternalURLs:
'http://__IP_INT__:8003': {}
+ Keepbalance:
+ InternalURLs:
+ 'http://__IP_INT__:9005': {}
Keepproxy:
ExternalURL: 'https://__HOSTNAME_EXT__:__KEEP_EXT_SSL_PORT__'
InternalURLs:
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
if grains.osfinger in ('CentOS Linux-7',) else
'/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
{%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
- if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+ if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
'/usr/bin/ruby' %}
### NGINX
---
# Copyright (C) The Arvados Authors. All rights reserved.
#
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
+ ### DEFAULT
+ arvados_workbench2_default.conf:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
arvados_workbench2_ssl.conf:
enabled: true
overwrite: true
#
# SPDX-License-Identifier: AGPL-3.0
+{%- if grains.os_family in ('RedHat',) %}
+ {%- set group = 'nginx' %}
+{%- else %}
+ {%- set group = 'www-data' %}
+{%- endif %}
+
### ARVADOS
arvados:
config:
- group: www-data
+ group: {{ group }}
### NGINX
nginx:
### SITES
servers:
managed:
+ ### DEFAULT
+ arvados_workbench_default.conf:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
arvados_workbench_ssl.conf:
enabled: true
overwrite: true
### POSTGRESQL
postgres:
+ # Centos-7's postgres package is too old, so we need to force using upstream's
+ # This is not required in Debian's family as they already ship with PG +11
+ {%- if salt['grains.get']('os_family') == 'RedHat' %}
+ use_upstream_repo: true
+ version: '12'
+
+ pkgs_deps:
+ - libicu
+ - libxslt
+ - systemd-sysv
+
+ pkgs_extra:
+ - postgresql12-contrib
+
+ {%- else %}
use_upstream_repo: false
pkgs_extra:
- postgresql-contrib
+ {%- endif %}
postgresconf: |-
listen_addresses = '*' # listen on all interfaces
+ # If you want to enable communications' encryption to the DB server,
+ # uncomment these entries
+ # ssl = on
+ # ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+ # ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
acls:
- ['local', 'all', 'postgres', 'peer']
- ['local', 'all', 'all', 'peer']
# Formulas versions
# ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
+# POSTGRES_TAG="v0.44.0"
# NGINX_TAG="v2.8.0"
-# DOCKER_TAG="v2.0.7"
+# DOCKER_TAG="v2.4.0"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
echo >&2 " controller"
echo >&2 " dispatcher"
echo >&2 " keepproxy"
+ echo >&2 " keepbalance"
echo >&2 " keepstore"
echo >&2 " keepweb"
echo >&2 " shell"
for i in ${2//,/ }
do
# Verify the role exists
- if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+ if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
echo "The role '${i}' is not a valid role"
usage
exit 1
# BRANCH="main"
# Other formula versions we depend on
-POSTGRES_TAG="v0.43.0"
+POSTGRES_TAG="v0.44.0"
NGINX_TAG="v2.8.0"
-DOCKER_TAG="v2.0.7"
+DOCKER_TAG="v2.4.0"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
# As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
- for c in controller websocket workbench workbench2 webshell keepweb keepproxy; do
- if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
- # Are we in a single-host-single-hostname env?
- CERT_NAME=${HOSTNAME_EXT}
- else
- # We are in a single-host-multiple-hostnames env
- CERT_NAME=${c}.${CLUSTER}.${DOMAIN}
- fi
-
- sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${CERT_NAME}*/g;
- s#__CERT_PEM__#/etc/letsencrypt/live/${CERT_NAME}/fullchain.pem#g;
- s#__CERT_KEY__#/etc/letsencrypt/live/${CERT_NAME}/privkey.pem#g" \
+ for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
+ sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${c}.${CLUSTER}.${DOMAIN}*/g;
+ s#__CERT_PEM__#/etc/letsencrypt/live/${c}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
+ s#__CERT_KEY__#/etc/letsencrypt/live/${c}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
${P_DIR}/nginx_${c}_configuration.sls
done
else
# Pillars
grep -q "docker" ${P_DIR}/top.sls || echo " - docker" >> ${P_DIR}/top.sls
;;
- "dispatcher")
- # States
- grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
- # Pillars
- # ATM, no specific pillar needed
- ;;
- "keepstore")
+ "dispatcher" | "keepbalance" | "keepstore")
# States
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars