Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>
end
def index
+ @objects = nil if !defined?(@objects)
find_objects_for_index if !@objects
render_index
end
end
def choose
+ @objects = nil if !defined?(@objects)
params[:limit] ||= 40
respond_to do |f|
if params[:partial]
def accept_uuid_as_id_param
- if params[:id] and params[:id].match /\D/
+ if params[:id] and params[:id].match(/\D/)
params[:uuid] = params.delete :id
end
end
helper_method :user_notifications
def user_notifications
+ @errors = nil if !defined?(@errors)
return [] if @errors or not current_user.andand.is_active or not Rails.configuration.show_user_notifications
@notifications ||= @@notification_tests.map do |t|
t.call(self, current_user)
helper_method :my_starred_projects
def my_starred_projects user
- return if @starred_projects
+ return if defined?(@starred_projects) && @starred_projects
links = Link.filter([['tail_uuid', '=', user.uuid],
['link_class', '=', 'star'],
['head_uuid', 'is_a', 'arvados#group']]).select(%w(head_uuid))
# That is: get toplevel projects under home, get subprojects of
# these projects, and so on until we hit the limit.
def my_wanted_projects(user, page_size=100)
- return @my_wanted_projects if @my_wanted_projects
+ return @my_wanted_projects if defined?(@my_wanted_projects) && @my_wanted_projects
from_top = []
uuids = [user.uuid]
end
def build_my_wanted_projects_tree(user, page_size=100)
- return @my_wanted_projects_tree if @my_wanted_projects_tree
+ return @my_wanted_projects_tree if defined?(@my_wanted_projects_tree) && @my_wanted_projects_tree
parent_of = {user.uuid => 'me'}
my_wanted_projects(user, page_size).each do |ob|
children_of[parent_of[ob.uuid]] ||= []
children_of[parent_of[ob.uuid]] << ob
end
- buildtree = lambda do |children_of, root_uuid=false|
+ buildtree = lambda do |chldrn_of, root_uuid=false|
tree = {}
- children_of[root_uuid].andand.each do |ob|
- tree[ob] = buildtree.call(children_of, ob.uuid)
+ chldrn_of[root_uuid].andand.each do |ob|
+ tree[ob] = buildtree.call(chldrn_of, ob.uuid)
end
tree
end
def remove_items
@removed_uuids = []
- links = []
params[:item_uuids].collect { |uuid| ArvadosBase.find uuid }.each do |item|
if item.class == Collection or item.class == Group
# Use delete API on collections and projects/groups
end
def current_api_host
- Rails.configuration.arvados_v1_base.gsub /https?:\/\/|\/arvados\/v1/,''
+ Rails.configuration.arvados_v1_base.gsub(/https?:\/\/|\/arvados\/v1/, '')
end
def current_uuid_prefix
def cwl_inputs_required(object, inputs_schema, set_attr_path)
r = 0
inputs_schema.each do |input|
- required, primary_type, param_id = cwl_input_info(input)
- dn, attrvalue = cwl_input_value(object, input, set_attr_path + [param_id])
+ required, _, param_id = cwl_input_info(input)
+ _, attrvalue = cwl_input_value(object, input, set_attr_path + [param_id])
r += 1 if required and attrvalue.nil?
end
r
private
def is_textile?( object, attr )
- is_textile = object.textile_attributes.andand.include?(attr)
+ object.textile_attributes.andand.include?(attr)
end
end
# For the benefit of themes that still expect $arvados_api_client to work:
class ArvadosClientProxyHack
def method_missing *args
- ArvadosApiClient.new_or_current.send *args
+ ArvadosApiClient.new_or_current.send(*args)
end
end
$arvados_api_client = ArvadosClientProxyHack.new
f0 = '' if f0 == '.'
f0 = f0[2..-1] if f0[0..1] == './'
f0 += '/' if not f0.empty?
- file_path = "#{f0}#{file[1]}"
+ "#{f0}#{file[1]}"
end
##
end
else
if step[:progress] and
- (re = step[:progress].match /^(\d+)\+(\d+)\/(\d+)$/)
+ (re = step[:progress].match(/^(\d+)\+(\d+)\/(\d+)$/))
pj[:progress] = (((re[1].to_f + re[2].to_f/2) / re[3].to_f) rescue 0.5)
else
pj[:progress] = 0.0
{label: 'output'})
end
# Input collection nodes
- output_pdhs = @opts[:output_collections].values.collect{|c|
- c[:portable_data_hash]}
+ output_pdhs = @opts[:output_collections].values.collect{|oc|
+ oc[:portable_data_hash]}
ProvenanceHelper::cr_input_pdhs(cr).each do |pdh|
if not output_pdhs.include?(pdh)
# Search for collections on the same project first
- cols = @opts[:input_collections][pdh].andand.select{|c|
- c[:owner_uuid] == cr[:owner_uuid]}
+ cols = @opts[:input_collections][pdh].andand.select{|ic|
+ ic[:owner_uuid] == cr[:owner_uuid]}
if not cols or cols.empty?
# Search for any collection with this PDH
cols = @opts[:input_collections][pdh]
end
def self.columns
+ @discovered_columns = [] if !defined?(@discovered_columns)
return @discovered_columns if @discovered_columns.andand.any?
- @discovered_columns = []
@attribute_info ||= {}
schema = arvados_api_client.discovery[:schemas][self.to_s.to_sym]
return @discovered_columns if schema.nil?
if opts[:class].is_a? Class
return opts[:class]
end
- if uuid.match /^[0-9a-f]{32}(\+[^,]+)*(,[0-9a-f]{32}(\+[^,]+)*)*$/
+ if uuid.match(/^[0-9a-f]{32}(\+[^,]+)*(,[0-9a-f]{32}(\+[^,]+)*)*$/)
return Collection
end
resource_class = nil
- uuid.match /^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/ do |re|
+ uuid.match(/^[0-9a-z]{5}-([0-9a-z]{5})-[0-9a-z]{15}$/) do |re|
resource_class ||= arvados_api_client.
kind_class(self.uuid_infix_object_kind[re[1]])
end
if opts[:referring_object] and
opts[:referring_attr] and
- opts[:referring_attr].match /_uuid$/
+ opts[:referring_attr].match(/_uuid$/)
resource_class ||= arvados_api_client.
kind_class(opts[:referring_object].
attributes[opts[:referring_attr].
@fetch_multiple_pages = true
@arvados_api_token = Thread.current[:arvados_api_token]
@reader_tokens = Thread.current[:reader_tokens]
+ @results = nil
+ @count = nil
+ @offset = 0
+ @cond = nil
+ @eager = nil
+ @select = nil
+ @orderby_spec = nil
+ @filters = nil
+ @distinct = nil
+ @include_trash = nil
+ @limit = nil
end
def eager(bool=true)
end
end
end
- @cond.keys.select { |x| x.match /_kind$/ }.each do |kind_key|
+ @cond.keys.select { |x| x.match(/_kind$/) }.each do |kind_key|
if @cond[kind_key].is_a? Class
@cond = @cond.merge({ kind_key => 'arvados#' + arvados_api_client.class_kind(@cond[kind_key]) })
end
def each(&block)
if not @results.nil?
- @results.each &block
+ @results.each(&block)
else
self.each_page do |items|
items.each do |i|
arvados_api_client.api "collections/#{self.uuid}/", "used_by"
end
- def uuid
- if self[:uuid].nil?
- return self[:portable_data_hash]
- else
- super
- end
- end
-
def friendly_link_name lookup=nil
name || portable_data_hash
end
@container = Container.find(container_uuid)
end
end
+ @container = nil if !defined?(@container)
@child_proxies = child_objects
end
} %>
</ul>
</li>
- <% if @name_link or @object %>
+ <% if (defined?(@name_link) && @name_link) or (defined?(@object) && @object) %>
<li class="nav-separator">
<i class="fa fa-lg fa-angle-double-right"></i>
</li>
<meta property="og:type" content="article" />
<meta property="og:url" content="<%= request.url %>" />
<meta property="og:site_name" content="<%= Rails.configuration.site_name %>" />
- <% if @object %>
+ <% if defined?(@object) && @object %>
<% if @object.respond_to?(:name) and @object.name.present? %>
<meta property="og:title" content="<%= @object.name%>" />
<% end %>
recent_cr_containers = recent_crs.map {|cr| cr.container_uuid}.compact.uniq
preload_objects_for_dataclass(Container, recent_cr_containers) if recent_cr_containers.andand.any?
- # fetch children of all the active crs in one call, if there are any
- active_crs = recent_crs.each {|cr| cr if (cr.priority.andand > 0 and cr.state != 'Final' and cr.container_uuid)}
-
wus = {}
outputs = []
recent_procs.each do |p|
# end
ActiveSupport::Inflector.inflections do |inflect|
- inflect.plural /^([Ss]pecimen)$/i, '\1s'
- inflect.singular /^([Ss]pecimen)s?/i, '\1'
- inflect.plural /^([Hh]uman)$/i, '\1s'
- inflect.singular /^([Hh]uman)s?/i, '\1'
+ inflect.plural(/^([Ss]pecimen)$/i, '\1s')
+ inflect.singular(/^([Ss]pecimen)s?/i, '\1')
+ inflect.plural(/^([Hh]uman)$/i, '\1s')
+ inflect.singular(/^([Hh]uman)s?/i, '\1')
end
%w(application.default application).each do |cfgfile|
path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
- if File.exists? path
+ if File.exist? path
yaml = ERB.new(IO.read path).result(binding)
confs = YAML.load(yaml, deserialize_symbols: true)
$application_config.merge!(confs['common'] || {})
# exception if not found. Use this with assertions to explain that
# the error signifies a failed test rather than an unexpected error
# during a testing procedure.
- def find? *args
+ def find?(*args)
begin
- find *args
+ find(*args)
rescue Capybara::ElementNotFound
false
end
def after_teardown
if self.class.want_reset_api_fixtures[:after_each_test] and
- @want_reset_api_fixtures != false
+ (!defined?(@want_reset_api_fixtures) or @want_reset_api_fixtures != false)
self.class.reset_api_fixtures_now
end
super
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3 && \
- /usr/local/rvm/bin/rvm-exec default gem install bundler && \
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \
/usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2
# Install golang binary
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.3 && \
- /usr/local/rvm/bin/rvm alias create default ruby-2.3
+ /usr/local/rvm/bin/rvm install 2.5 && \
+ /usr/local/rvm/bin/rvm alias create default ruby-2.5
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
WORKSPACE=/path/to/arvados $(basename $0) [options]
--target <target>
- Distribution to build packages for (default: debian8)
+ Distribution to build packages for (default: debian9)
--command
Build command to execute (default: use built-in Docker image command)
--test-packages
exit 1
fi
-TARGET=debian8
+TARGET=debian9
COMMAND=
DEBUG=
set +e
mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null
set -e
+set -x
# Build packages. ulimit option can be removed when debian8 and ubuntu1404 are retired
if docker run --ulimit nofile=4096:4096 \
--rm \
--debug
Output debug information (default: false)
--target
- Distribution to build packages for (default: debian8)
+ Distribution to build packages for (default: debian9)
WORKSPACE=path Path to the Arvados SSO source tree to build packages from
EXITCODE=0
DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian8
+TARGET=debian9
PARSEDOPTS=$(getopt --name "$0" --longoptions \
help,build-bundle-packages,debug,target: \
WORKSPACE=/path/to/arvados $(basename $0) [options]
--target <target>
- Distribution to build packages for (default: debian8)
+ Distribution to build packages for (default: debian9)
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
exit 1
fi
-TARGET=debian8
+TARGET=debian9
UPLOAD=0
RC=0
title "End of upload packages (`timer`)"
fi
-exit_cleanly
\ No newline at end of file
+exit_cleanly
&& cd "$WORKSPACE/$srcpath" \
&& bundle_install_trylocal \
&& gem build "$gemname.gemspec" \
- && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+ && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1)
}
install_sdk/ruby() {
testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/"
done
+testfuncargs["sdk/cli"]="sdk/cli"
+
if [[ -z ${interactive} ]]; then
install_all
test_all
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Ruby 2.3 is recommended; Ruby 2.1 is also known to work.
+Ruby 2.5 is recommended; Ruby 2.3 is also known to work.
h4(#rvm). *Option 1: Install with RVM*
<notextile>
-<pre><code><span class="userinput">sudo gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
-\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.3
+<pre><code><span class="userinput">sudo gpg --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.5
</span></code></pre></notextile>
Either log out and log back in to activate RVM, or explicitly load it in all open shells like this:
<notextile>
<pre><code><span class="userinput">mkdir -p ~/src
cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz | tar xz
-cd ruby-2.3.3
+curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.5.tar.gz | tar xz
+cd ruby-2.5.5
./configure --disable-install-rdoc
make
sudo make install
--- /dev/null
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Ruby 2.3 is recommended; Ruby 2.1 is also known to work.
+
+h4(#rvm). *Option 1: Install with RVM*
+
+<notextile>
+<pre><code><span class="userinput">sudo gpg --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | sudo bash -s stable --ruby=2.3
+</span></code></pre></notextile>
+
+Either log out and log back in to activate RVM, or explicitly load it in all open shells like this:
+
+<notextile>
+<pre><code><span class="userinput">source /usr/local/rvm/scripts/rvm
+</span></code></pre></notextile>
+
+Once RVM is activated in your shell, install Bundler:
+
+<notextile>
+<pre><code>~$ <span class="userinput">gem install bundler</span>
+</code></pre></notextile>
+
+h4(#fromsource). *Option 2: Install from source*
+
+Install prerequisites for Debian 8:
+
+<notextile>
+<pre><code><span class="userinput">sudo apt-get install \
+ bison build-essential gettext libcurl3 libcurl3-gnutls \
+ libcurl4-openssl-dev libpcre3-dev libreadline-dev \
+ libssl-dev libxslt1.1 zlib1g-dev
+</span></code></pre></notextile>
+
+Install prerequisites for CentOS 7:
+
+<notextile>
+<pre><code><span class="userinput">sudo yum install \
+ libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel \
+ patch readline-devel zlib-devel libffi-devel openssl-devel \
+ make automake libtool bison sqlite-devel tar
+</span></code></pre></notextile>
+
+Install prerequisites for Ubuntu 12.04 or 14.04:
+
+<notextile>
+<pre><code><span class="userinput">sudo apt-get install \
+ gawk g++ gcc make libc6-dev libreadline6-dev zlib1g-dev libssl-dev \
+ libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev \
+ libncurses5-dev automake libtool bison pkg-config libffi-dev curl
+</span></code></pre></notextile>
+
+Build and install Ruby:
+
+<notextile>
+<pre><code><span class="userinput">mkdir -p ~/src
+cd ~/src
+curl -f http://cache.ruby-lang.org/pub/ruby/2.3/ruby-2.3.3.tar.gz | tar xz
+cd ruby-2.3.3
+./configure --disable-install-rdoc
+make
+sudo make install
+
+sudo -i gem install bundler</span>
+</code></pre></notextile>
TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
{% endcomment %}
-h3. v1.4.0 (2019-06-05)
+table(table table-bordered table-condensed).
+|_. development|"master":#master|\3.|
+|_. latest stable|"v1.4.0":#v1_4_0|\3.|
+|_\5. past stable|
+|"v1.3.3":#v1_3_3|"v1.3.0":#v1_3_0|\3.|
+|"v1.2.1":#v1_2_1|"v1.2.0":#v1_2_0|\3.|
+|"v1.1.4":#v1_1_4|"v1.1.3":#v1_1_3|"v1.1.2":#v1_1_2|"v1.1.1":#v1_1_1|"v1.1.0":#v1_1_0|
+|\5. "older":#older|
+
+h3(#master). development master (as of 2019-06-07)
+
+h4. No longer stripping ':' from strings in serialized database columns
+
+ (bug #15311) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record. This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'. Unfortunately this corrupted fields where the leading ":" was intentional. This behavior has been removed.
+
+You can test if any records in your database are affected by going to the API server directory and running @bundle exec rake symbols:check@. This will report which records contain fields with a leading ':' that would previously have been stripped. If there are records to be updated, you can update the database using @bundle exec rake symbols:stringify@.
+
+h3(#v1_4_0). v1.4.0 (2019-06-05)
h4. Populating the new file_count and file_size_total columns on the collections table
Arvados is migrating to a centralized configuration file for all components. During the migration, legacy configuration files will continue to be loaded. See "Migrating Configuration":config-migration.html for details.
-h3. v1.3.0 (2018-12-05)
+h3(#v1_3_3). v1.3.3 (2019-05-14)
+
+This release corrects a potential data loss issue, if you are running Arvados 1.3.0 or 1.3.1 we strongly recommended disabling @keep-balance@ until you can upgrade to 1.3.3 or 1.4.0. With keep-balance disabled, there is no chance of data loss.
+
+We've put together a "wiki page":https://dev.arvados.org/projects/arvados/wiki/Recovering_lost_data which outlines how to recover blocks which have been put in the trash, but not yet deleted, as well as how to identify any collections which have missing blocks so that they can be regenerated. The keep-balance component has been enhanced to provide a list of missing blocks and affected collections and we've provided a "utility script":https://github.com/curoverse/arvados/blob/master/tools/keep-xref/keep-xref.py which can be used to identify the workflows that generated those collections and who ran those workflows, so that they can be rerun.
+
+h3(#v1_3_0). v1.3.0 (2018-12-05)
This release includes several database migrations, which will be executed automatically as part of the API server upgrade. On large Arvados installations, these migrations will take a while. We've seen the upgrade take 30 minutes or more on installations with a lot of collections.
Support for the deprecated "jobs" API is broken in this release. Users who rely on it should not upgrade. This will be fixed in an upcoming 1.3.1 patch release, however users are "encouraged to migrate":upgrade-crunch2.html as support for the "jobs" API will be dropped in an upcoming release. Users who are already using the "containers" API are not affected.
-h3. v1.2.1 (2018-11-26)
+h3(#v1_2_1). v1.2.1 (2018-11-26)
There are no special upgrade notes for this release.
-h3. v1.2.0 (2018-09-05)
+h3(#v1_2_0). v1.2.0 (2018-09-05)
h4. Regenerate Postgres table statistics
Verify your setup by confirming that API calls appear in the controller's logs (_e.g._, @journalctl -fu arvados-controller@) while loading a workbench page.
-h3. v1.1.4 (2018-04-10)
+h3(#v1_1_4). v1.1.4 (2018-04-10)
h4. arvados-cwl-runner regressions (2018-04-05)
This bug has been fixed in Arvados release v1.2.0.
-h3. v1.1.3 (2018-02-08)
+h3(#v1_1_3). v1.1.3 (2018-02-08)
There are no special upgrade notes for this release.
-h3. v1.1.2 (2017-12-22)
+h3(#v1_1_2). v1.1.2 (2017-12-22)
h4. The minimum version for Postgres is now 9.4 (2017-12-08)
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. v1.1.1 (2017-11-30)
+h3(#v1_1_1). v1.1.1 (2017-11-30)
There are no special upgrade notes for this release.
-h3. v1.1.0 (2017-10-24)
+h3(#v1_1_0). v1.1.0 (2017-10-24)
h4. The minimum version for Postgres is now 9.3 (2017-09-25)
*# Install the @rh-postgresql94@ backport package from either Software Collections: http://doc.arvados.org/install/install-postgresql.html or the Postgres developers: https://www.postgresql.org/download/linux/redhat/
*# Restore from the backup using @psql@
-h3. Older versions
+h3(#older). Older versions
h4. Upgrade slower than usual (2017-06-30)
h3(#install_ruby_and_bundler). Install Ruby and Bundler
-{% include 'install_ruby_and_bundler' %}
+{% include 'install_ruby_and_bundler_sso' %}
h3(#install_web_server). Set up a Web server
end
def test_output_collection_owner_uuid
+ skip "Depends on a post 1.3 python-arvados-python-client package being installed"
+
j = jobspec :grep_local
out, err = capture_subprocess_io do
tryjobrecord j, binstubs: ['arv-mount', 'output_coll_owner']
if pattern is None:
continue
sfpath = substitute(primary["location"], pattern)
- required = builder.do_eval(sf["required"], context=primary)
+ required = builder.do_eval(sf.get("required"), context=primary)
if fsaccess.exists(sfpath):
primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20190603140227',
+ 'cwltool==1.0.20190607183319',
'schema-salad==4.2.20190417121603',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
cwltool: "http://commonwl.org/cwltool#"
requirements:
SubworkflowFeatureRequirement: {}
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
inputs:
i:
type: File
arv: "http://arvados.org/cwl#"
requirements:
SubworkflowFeatureRequirement: {}
+hints:
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
step1:
requirements:
arv:RunInSingleContainer: {}
in: []
out: []
- run: default-dir6.cwl
\ No newline at end of file
+ run: default-dir6.cwl
arv: "http://arvados.org/cwl#"
requirements:
SubworkflowFeatureRequirement: {}
+hints:
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
step1:
requirements:
arv:RunInSingleContainer: {}
in: []
out: []
- run: default-dir7.cwl
\ No newline at end of file
+ run: default-dir7.cwl
ScatterFeatureRequirement: {}
InlineJavascriptRequirement: {}
StepInputExpressionRequirement: {}
+hints:
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
substep:
in:
ScatterFeatureRequirement: {}
InlineJavascriptRequirement: {}
StepInputExpressionRequirement: {}
+hints:
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
substep:
in:
ScatterFeatureRequirement: {}
InlineJavascriptRequirement: {}
StepInputExpressionRequirement: {}
+hints:
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
substep:
in:
out: [out]
hints:
- class: arv:RunInSingleContainer
+ - class: DockerRequirement
+ dockerPull: arvados/jobs:1.4.0.20190604172024
run:
class: Workflow
id: mysub
out:
type: string
outputBinding:
- outputEval: "out"
+ outputEval: $("out")
baseCommand: cat
hints:
arv:IntermediateOutput:
outputTTL: 60
+ DockerRequirement:
+ dockerPull: arvados/jobs:1.4.0.20190604172024
steps:
substep:
in:
out:
type: string
outputBinding:
- outputEval: "out"
+ outputEval: $("out")
baseCommand: cat
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
nil,
map[string]string{
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
)
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
uglifier (~> 2.0)
BUNDLED WITH
- 1.16.6
+ 1.17.3
conditions[0] << " and #{ar_table_name}.#{attr} in (?)"
conditions << value
end
- elsif value.is_a? String or value.is_a? Fixnum or value == true or value == false
+ elsif value.is_a? String or value.is_a? Integer or value == true or value == false
conditions[0] << " and #{ar_table_name}.#{attr}=?"
conditions << value
elsif value.is_a? Hash
after_create :log_create
after_update :log_update
after_destroy :log_destroy
- after_find :convert_serialized_symbols_to_strings
before_validation :normalize_collection_uuids
before_validation :set_default_owner
validate :ensure_valid_uuids
false
end
- def self.has_symbols? x
- if x.is_a? Hash
- x.each do |k,v|
- return true if has_symbols?(k) or has_symbols?(v)
- end
- elsif x.is_a? Array
- x.each do |k|
- return true if has_symbols?(k)
- end
- elsif x.is_a? Symbol
- return true
- elsif x.is_a? String
- return true if x.start_with?(':') && !x.start_with?('::')
- end
- false
- end
-
- def self.recursive_stringify x
- if x.is_a? Hash
- Hash[x.collect do |k,v|
- [recursive_stringify(k), recursive_stringify(v)]
- end]
- elsif x.is_a? Array
- x.collect do |k|
- recursive_stringify k
- end
- elsif x.is_a? Symbol
- x.to_s
- elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
- x[1..-1]
- else
- x
- end
- end
-
def self.where_serialized(colname, value, md5: false)
colsql = colname.to_s
if md5
self.class.serialized_attributes
end
- def convert_serialized_symbols_to_strings
- # ensure_serialized_attribute_type should prevent symbols from
- # getting into the database in the first place. If someone managed
- # to get them into the database (perhaps using an older version)
- # we'll convert symbols to strings when loading from the
- # database. (Otherwise, loading and saving an object with existing
- # symbols in a serialized field will crash.)
- jsonb_cols = self.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
- (jsonb_cols + self.class.serialized_attributes.keys).uniq.each do |colname|
- if self.class.has_symbols? attributes[colname]
- attributes[colname] = self.class.recursive_stringify attributes[colname]
- send(colname + '=',
- self.class.recursive_stringify(attributes[colname]))
- end
- end
- end
-
def foreign_key_attributes
attributes.keys.select { |a| a.match(/_uuid$/) }
end
begin
must_git gitdir, "branch"
rescue GitError => e
- raise unless /Not a git repository/ =~ e.to_s
+ raise unless /Not a git repository/i =~ e.to_s
# OK, this just means we need to create a blank cache repository
# before fetching.
FileUtils.mkdir_p gitdir
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'current_api_client'
+
+include CurrentApiClient
+
+def has_symbols? x
+ if x.is_a? Hash
+ x.each do |k,v|
+ return true if has_symbols?(k) or has_symbols?(v)
+ end
+ elsif x.is_a? Array
+ x.each do |k|
+ return true if has_symbols?(k)
+ end
+ elsif x.is_a? Symbol
+ return true
+ elsif x.is_a? String
+ return true if x.start_with?(':') && !x.start_with?('::')
+ end
+ false
+end
+
+def check_for_serialized_symbols rec
+ jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
+ (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
+ if has_symbols? rec.attributes[colname]
+ st = recursive_stringify rec.attributes[colname]
+ puts "Found value potentially containing Ruby symbols in #{colname} attribute of #{rec.uuid}, current value is\n#{rec.attributes[colname].to_s[0..1024]}\nrake symbols:stringify will update it to:\n#{st.to_s[0..1024]}\n\n"
+ end
+ end
+end
+
+def recursive_stringify x
+ if x.is_a? Hash
+ Hash[x.collect do |k,v|
+ [recursive_stringify(k), recursive_stringify(v)]
+ end]
+ elsif x.is_a? Array
+ x.collect do |k|
+ recursive_stringify k
+ end
+ elsif x.is_a? Symbol
+ x.to_s
+ elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
+ x[1..-1]
+ else
+ x
+ end
+end
+
+def stringify_serialized_symbols rec
+ # ensure_serialized_attribute_type should prevent symbols from
+ # getting into the database in the first place. If someone managed
+ # to get them into the database (perhaps using an older version)
+ # we'll convert symbols to strings when loading from the
+ # database. (Otherwise, loading and saving an object with existing
+ # symbols in a serialized field will crash.)
+ jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
+ (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
+ if has_symbols? rec.attributes[colname]
+ begin
+ st = recursive_stringify rec.attributes[colname]
+ puts "Updating #{colname} attribute of #{rec.uuid} from\n#{rec.attributes[colname].to_s[0..1024]}\nto\n#{st.to_s[0..1024]}\n\n"
+ rec.write_attribute(colname, st)
+ rec.save!
+ rescue => e
+ puts "Failed to update #{rec.uuid}: #{e}"
+ end
+ end
+ end
+end
+
+namespace :symbols do
+ desc 'Warn about serialized values starting with ":" that may be symbols'
+ task check: :environment do
+ [ApiClientAuthorization, ApiClient,
+ AuthorizedKey, Collection,
+ Container, ContainerRequest, Group,
+ Human, Job, JobTask, KeepDisk, KeepService, Link,
+ Node, PipelineInstance, PipelineTemplate,
+ Repository, Specimen, Trait, User, VirtualMachine,
+ Workflow].each do |klass|
+ act_as_system_user do
+ klass.all.each do |c|
+ check_for_serialized_symbols c
+ end
+ end
+ end
+ end
+
+ task stringify: :environment do
+ [ApiClientAuthorization, ApiClient,
+ AuthorizedKey, Collection,
+ Container, ContainerRequest, Group,
+ Human, Job, JobTask, KeepDisk, KeepService, Link,
+ Node, PipelineInstance, PipelineTemplate,
+ Repository, Specimen, Trait, User, VirtualMachine,
+ Workflow].each do |klass|
+ act_as_system_user do
+ klass.all.each do |c|
+ stringify_serialized_symbols c
+ end
+ end
+ end
+ end
+end
head_uuid: zzzzz-4zz18-pyw8yp9g3pr7irn
properties: {}
+#
+# This fixture was used in the test "Stringify symbols coming from
+# serialized attribute in database" which tested the hook
+# "convert_serialized_symbols_to_strings". That hook (and the
+# corresponding test) was removed in #15311. This fixture remains to
+# facilitate manual testing of the "rake symbols:check" and "rake
+# symbols:stringify" tasks that we added to assist with database
+# fixup.
+#
has_symbol_keys_in_database_somehow:
uuid: zzzzz-o0j2j-enl1wg58310loc6
owner_uuid: zzzzz-tpzed-000000000000000
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'test_helper'
+
+class ContainerRequestIntegrationTest < ActionDispatch::IntegrationTest
+
+ test "test colon in input" do
+ # Tests for bug #15311 where strings with leading colons get
+ # corrupted when the leading ":" is stripped.
+ val = {"itemSeparator" => ":"}
+ post "/arvados/v1/container_requests",
+ params: {
+ :container_request => {
+ :name => "workflow",
+ :state => "Committed",
+ :command => ["echo"],
+ :container_image => "fa3c1a9cb6783f85f2ecda037e07b8c3+167",
+ :output_path => "/",
+ :priority => 1,
+ :runtime_constraints => {"vcpus" => 1, "ram" => 1},
+ :mounts => {
+ :foo => {
+ :kind => "json",
+ :content => JSON.parse(SafeJSON.dump(val)),
+ }
+ }
+ }
+ }.to_json,
+ headers: {
+ 'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active).api_token}",
+ 'CONTENT_TYPE' => 'application/json'
+ }
+ assert_response :success
+ assert_equal "arvados#containerRequest", json_response['kind']
+ assert_equal val, json_response['mounts']['foo']['content']
+ end
+end
end
end
- test "Stringify symbols coming from serialized attribute in database" do
- set_user_from_auth :admin_trustedclient
- fixed = Link.find_by_uuid(links(:has_symbol_keys_in_database_somehow).uuid)
- assert_equal(["baz", "foo"], fixed.properties.keys.sort,
- "Hash symbol keys from DB did not get stringified.")
- assert_equal(['waz', 'waz', 'waz', 1, nil, false, true],
- fixed.properties['baz'],
- "Array symbol values from DB did not get stringified.")
- assert_equal true, fixed.save, "Failed to save fixed model back to db."
- end
-
test "No HashWithIndifferentAccess in database" do
set_user_from_auth :admin_trustedclient
link = Link.create!(link_class: 'test',
--- /dev/null
+PATH
+ remote: .
+ specs:
+ arvados-login-sync (1.3.3.20190528194843)
+ arvados (~> 1.3.0, >= 1.3.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (5.0.7.2)
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ i18n (>= 0.7, < 2)
+ minitest (~> 5.1)
+ tzinfo (~> 1.1)
+ addressable (2.6.0)
+ public_suffix (>= 2.0.2, < 4.0)
+ andand (1.3.3)
+ arvados (1.3.3.20190320201707)
+ activesupport (>= 3)
+ andand (~> 1.3, >= 1.3.3)
+ arvados-google-api-client (>= 0.7, < 0.8.9)
+ i18n (~> 0)
+ json (>= 1.7.7, < 3)
+ jwt (>= 0.1.5, < 2)
+ arvados-google-api-client (0.8.7.2)
+ activesupport (>= 3.2, < 5.1)
+ addressable (~> 2.3)
+ autoparse (~> 0.3)
+ extlib (~> 0.9)
+ faraday (~> 0.9)
+ googleauth (~> 0.3)
+ launchy (~> 2.4)
+ multi_json (~> 1.10)
+ retriable (~> 1.4)
+ signet (~> 0.6)
+ autoparse (0.3.3)
+ addressable (>= 2.3.1)
+ extlib (>= 0.9.15)
+ multi_json (>= 1.0.0)
+ concurrent-ruby (1.1.5)
+ extlib (0.9.16)
+ faraday (0.15.4)
+ multipart-post (>= 1.2, < 3)
+ googleauth (0.8.1)
+ faraday (~> 0.12)
+ jwt (>= 1.4, < 3.0)
+ memoist (~> 0.16)
+ multi_json (~> 1.11)
+ os (>= 0.9, < 2.0)
+ signet (~> 0.7)
+ i18n (0.9.5)
+ concurrent-ruby (~> 1.0)
+ json (2.2.0)
+ jwt (1.5.6)
+ launchy (2.4.3)
+ addressable (~> 2.3)
+ memoist (0.16.0)
+ metaclass (0.0.4)
+ minitest (5.11.3)
+ mocha (1.8.0)
+ metaclass (~> 0.0.1)
+ multi_json (1.13.1)
+ multipart-post (2.1.1)
+ os (1.0.1)
+ public_suffix (3.0.3)
+ rake (12.3.2)
+ retriable (1.4.1)
+ signet (0.11.0)
+ addressable (~> 2.3)
+ faraday (~> 0.9)
+ jwt (>= 1.5, < 3.0)
+ multi_json (~> 1.10)
+ thread_safe (0.3.6)
+ tzinfo (1.2.5)
+ thread_safe (~> 0.1)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ arvados-login-sync!
+ minitest (>= 5.0.0)
+ mocha (>= 1.5.0)
+ rake
+
+BUNDLED WITH
+ 1.17.3
-set_serial $RANDOM$RANDOM \
-extfile <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nkeyUsage=critical,digitalSignature,keyEncipherment\nsubjectAltName=DNS:localhost,$san")) \
- -extensions x509_ext
+ -extensions x509_ext \
+ -days 365
chown arvbox:arvbox /var/lib/arvados/server-cert-${localip}.*
fi
projects / arvados.git / commitdiff