RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9
# Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> /etc/gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.5 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
+ /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
# udev daemon can't start in a container, so don't try.
arv-put --version >/dev/null
-/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+PYTHON=`ls /usr/share/python3*/dist/python3-arvados-python-client/bin/python3 |head -n1`
+
+$PYTHON << EOF
import arvados
print("Successfully imported arvados")
EOF
Build only a specific package (or ONLY_BUILD from environment)
--arch <arch>
Build a specific architecture (or ARCH from environment, defaults to native architecture)
+--force-build
+ Build even if the package exists upstream or if it has already been
+ built locally
+--force-test
+ Test even if there is no new untested package
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,rc,target:,only-build:,arch:,build-version: \
+ help,debug,upload,rc,target:,force-test,only-build:,force-build,arch:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --force-test)
+ FORCE_TEST=1
+ ;;
+ --force-build)
+ FORCE_BUILD=1
+ ;;
--only-build)
ONLY_BUILD="$2"; shift
;;
build_args+=(--only-build "$ONLY_BUILD")
fi
+if [[ -n "$FORCE_BUILD" ]]; then
+ build_args+=(--force-build)
+fi
+
+if [[ -n "$FORCE_TEST" ]]; then
+ build_args+=(--force-test)
+fi
+
if [[ -n "$ARCH" ]]; then
build_args+=(--arch "$ARCH")
fi
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.5 or newer is required.
+Ruby 2.6 or newer is required.
* "Option 1: Install from packages":#packages
* "Option 2: Install with RVM":#rvm
h3. Centos 7
-The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install Ruby 2.5 or later.
+The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
h3. Debian and Ubuntu
-Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5 or newer, which is sufficient for Arvados.
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+
+Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
<notextile>
<pre><code># <span class="userinput">apt-get --no-install-recommends install ruby ruby-dev</span></code></pre>
<notextile>
<pre><code># <span class="userinput">gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
</span></code></pre></notextile>
-This command installs the latest Ruby 2.5.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
To use Ruby installed from RVM, load it in an open shell like this:
<notextile>
<pre><code><span class="userinput">mkdir -p ~/src
cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.8.tar.gz | tar xz
-cd ruby-2.5.8
+curl -f https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz | tar xz
+cd ruby-2.7.5
./configure --disable-install-rdoc
make
sudo make install
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2022-02-10)
+h2(#main). development main (as of 2022-03-08)
"previous: Upgrading to 2.3.0":#v2_3_0
+h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
+
+Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
+
+This means that they are now installed under @/usr/share/python3.8@ (before, the path was @/usr/share/python3@). If you rely on the @python3@ executable from the packages (e.g. to load a virtualenv), you may need to update the path to that executable.
+
+h3. Minimum supported Ruby version is now 2.6
+
+The minimum supported Ruby version is now 2.6. If you are running Arvados on Debian 10 or Ubuntu 18.04, you may need to switch to using RVM or upgrade your OS. See "Install Ruby and Bundler":../install/ruby.html for more information.
+
h3. Anonymous token changes
The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary. If the anonymous token in @config.yml@ is specified as a full V2 token, that will now generate a warning - it should be updated to list just the secret (i.e. the part after the last forward slash).
h2(#supportedlinux). Supported GNU/Linux distributions
table(table table-bordered table-condensed).
-|_. Distribution|_. State|_. Last supported version|
+|_. Distribution|_. State|_. Last supported Arvados version|
|CentOS 7|Supported|Latest|
|Debian 11 ("bullseye")|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
+
<notextile>
<pre>~$ <code class="userinput">source /usr/share/python3/dist/python3-arvados-python-client/bin/activate</code>
(python-arvados-python-client) ~$ <code class="userinput">python</code>
RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version
# use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python
-RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
+RUN PYTHON=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python|head -n1` && rm -f /usr/bin/python && ln -s $PYTHON /usr/bin/python
+RUN PYTHON3=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python3|head -n1` && rm -f /usr/bin/python3 && ln -s $PYTHON3 /usr/bin/python3
# Install dependencies and set up system.
RUN /usr/sbin/adduser --disabled-password \
import unittest
import os
import functools
+import threading
import cwltool.process
import cwltool.secrets
import cwltool.load_tool
"basedir": "",
"make_fs_access": make_fs_access,
"construct_tool_object": runner.arv_make_tool,
- "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
+ "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
+ "loader": Loader({}),
+ "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"name": "test_run_"+str(enable_reuse),
"make_fs_access": make_fs_access,
"tmpdir": "/tmp",
+ "outdir": "/tmp",
"enable_reuse": enable_reuse,
"priority": 500,
- "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+ "workflow_eval_lock": threading.Condition(threading.RLock())
})
if isinstance(runner, mock.MagicMock):
"baseCommand": "echo",
"arguments": [],
"id": "",
- "cwlVersion": "v1.2",
- "class": "CommandLineTool"
+ "cwlVersion": "v1.0",
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner, True)
- arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
arvtool.formatgraph = None
container_request = {
'name': 'test_run_True',
'runtime_constraints': {
'vcpus': 1,
- 'ram': 268435456
+ 'ram': 1073741824,
},
'use_existing': True,
'priority': 500,
projects / arvados.git / commitdiff