.Rproj.user
_version.py
*.bak
+arvados-snakeoil-ca.pem
+.vagrant
sdk/cwl/tests/wf/feddemo
go.mod
go.sum
+sdk/python/tests/fed-migrate/CWLFile
+sdk/python/tests/fed-migrate/*.cwl
+sdk/python/tests/fed-migrate/*.cwlex
+doc/install/*.xlsx
Visit [Hacking Arvados](https://dev.arvados.org/projects/arvados/wiki/Hacking) for
detailed information about setting up an Arvados development
-environment, development process, coding standards, and notes about specific components.
+environment, development process, [coding standards](https://dev.arvados.org/projects/arvados/wiki/Coding_Standards), and notes about specific components.
If you wish to build the Arvados documentation from a local git clone, see
[doc/README.textile](doc/README.textile) for instructions.
if current_user && !profile_config.empty?
current_user_profile = current_user.prefs[:profile]
profile_config.each do |k, entry|
- if entry['Required']
+ if entry[:Required]
if !current_user_profile ||
!current_user_profile[k] ||
current_user_profile[k].empty?
end
end
params[:merge] = true
+
+ if !@updates[:reuse_steps].nil?
+ if @updates[:reuse_steps] == "false"
+ @updates[:reuse_steps] = false
+ end
+ @updates[:command] ||= @object.command
+ @updates[:command] -= ["--disable-reuse", "--enable-reuse"]
+ if @updates[:reuse_steps]
+ @updates[:command].insert(1, "--enable-reuse")
+ else
+ @updates[:command].insert(1, "--disable-reuse")
+ end
+ @updates.delete(:reuse_steps)
+ end
+
begin
super
rescue => e
@object = ContainerRequest.new
- # By default the copied CR won't be reusing containers, unless use_existing=true
- # param is passed.
+ # set owner_uuid to that of source, provided it is a project and writable by current user
+ if params[:work_unit].andand[:owner_uuid]
+ @object.owner_uuid = src.owner_uuid = params[:work_unit][:owner_uuid]
+ else
+ current_project = Group.find(src.owner_uuid) rescue nil
+ if (current_project && current_project.writable_by.andand.include?(current_user.uuid))
+ @object.owner_uuid = src.owner_uuid
+ end
+ end
+
command = src.command
- if params[:use_existing]
- @object.use_existing = true
+ if command[0] == 'arvados-cwl-runner'
+ command.each_with_index do |arg, i|
+ if arg.start_with? "--project-uuid="
+ command[i] = "--project-uuid=#{@object.owner_uuid}"
+ end
+ end
+ command -= ["--disable-reuse", "--enable-reuse"]
+ command.insert(1, '--enable-reuse')
+ end
+
+ if params[:use_existing] == "false"
+ params[:use_existing] = false
+ elsif params[:use_existing] == "true"
+ params[:use_existing] = true
+ end
+
+ if params[:use_existing] || params[:use_existing].nil?
+ # If nil, reuse workflow steps but not the workflow runner.
+ @object.use_existing = !!params[:use_existing]
+
# Pass the correct argument to arvados-cwl-runner command.
- if src.command[0] == 'arvados-cwl-runner'
- command = src.command - ['--disable-reuse']
+ if command[0] == 'arvados-cwl-runner'
+ command -= ["--disable-reuse", "--enable-reuse"]
command.insert(1, '--enable-reuse')
end
else
@object.use_existing = false
# Pass the correct argument to arvados-cwl-runner command.
- if src.command[0] == 'arvados-cwl-runner'
- command = src.command - ['--enable-reuse']
+ if command[0] == 'arvados-cwl-runner'
+ command -= ["--disable-reuse", "--enable-reuse"]
command.insert(1, '--disable-reuse')
end
end
@object.scheduling_parameters = src.scheduling_parameters
@object.state = 'Uncommitted'
- # set owner_uuid to that of source, provided it is a project and writable by current user
- current_project = Group.find(src.owner_uuid) rescue nil
- if (current_project && current_project.writable_by.andand.include?(current_user.uuid))
- @object.owner_uuid = src.owner_uuid
- end
-
super
end
owner_uuids = @objects.collect(&:owner_uuid).uniq
@owners = {}
@not_trashed = {}
- Group.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp|
- @owners[grp.uuid] = grp
- end
- User.filter([["uuid", "in", owner_uuids]]).with_count("none").include_trash(true).each do |grp|
- @owners[grp.uuid] = grp
- @not_trashed[grp.uuid] = true
+ [Group, User].each do |owner_class|
+ owner_class.filter([["uuid", "in", owner_uuids]]).with_count("none")
+ .include_trash(true).fetch_multiple_pages(false)
+ .each do |owner|
+ @owners[owner.uuid] = owner
+ end
end
Group.filter([["uuid", "in", owner_uuids]]).with_count("none").select([:uuid]).each do |grp|
@not_trashed[grp.uuid] = true
def profile
params[:offer_return_to] ||= params[:return_to]
+
+ # In a federation situation, when you get a user record using
+ # "current user of token" it can fetch a stale user record from
+ # the local cluster. So even if profile settings were just written
+ # to the user record on the login cluster (because the user just
+ # filled out the profile), those profile settings may not appear
+ # in the "current user" response because it is returning a cached
+ # record from the local cluster.
+ #
+ # In this case, explicitly fetching user record forces it to get a
+ # fresh record from the login cluster.
+ Thread.current[:user] = User.find(current_user.uuid)
end
def activity
end
attrs['command'] = ["arvados-cwl-runner",
+ "--enable-reuse",
"--local",
"--api=containers",
"--project-uuid=#{params['work_unit']['owner_uuid']}",
end
input_type = 'text'
+ opt_selection = nil
attrtype = object.class.attribute_info[attr.to_sym].andand[:type]
if attrtype == 'text' or attr == 'description'
input_type = 'textarea'
elsif attrtype == 'datetime'
input_type = 'date'
+ elsif attrtype == 'boolean'
+ input_type = 'select'
+ opt_selection = ([{value: "true", text: "true"}, {value: "false", text: "false"}]).to_json
else
input_type = 'text'
end
"data-emptytext" => '(none)',
"data-placement" => "bottom",
"data-type" => input_type,
+ "data-source" => opt_selection,
"data-title" => "Edit #{attr.to_s.gsub '_', ' '}",
"data-name" => htmloptions['selection_name'] || attr,
"data-object-uuid" => object.uuid,
true
end
+ def self.copies_to_projects?
+ false
+ end
+
def work_unit(label=nil, child_objects=nil)
ContainerWorkUnit.new(self, label, self.uuid, child_objects=child_objects)
end
+
+ def editable_attributes
+ super + ["reuse_steps"]
+ end
+
+ def reuse_steps
+ command.each do |arg|
+ if arg == "--enable-reuse"
+ return true
+ end
+ end
+ false
+ end
+
+ def self.attribute_info
+ self.columns
+ @attribute_info[:reuse_steps] = {:type => "boolean"}
+ @attribute_info
+ end
+
end
end
def self.creatable?
- current_user and current_user.is_admin
+ current_user.andand.is_admin
end
end
}
</script>
- <%= link_to raw('<i class="fa fa-fw fa-play"></i> Re-run...'),
- "#",
- {class: 'btn btn-sm btn-primary', 'data-toggle' => 'modal',
- 'data-target' => '#clone-and-edit-modal-window',
- title: 'This will make a copy and take you there. You can then make any needed changes and run it'} %>
-
-<div id="clone-and-edit-modal-window" class="modal fade" role="dialog"
- aria-labelledby="myModalLabel" aria-hidden="true">
- <div class="modal-dialog">
- <div class="modal-content">
-
- <%= form_tag copy_container_request_path do |f| %>
-
- <div class="modal-header">
- <button type="button" class="close" onClick="reset_form_cr_reuse()" data-dismiss="modal" aria-hidden="true">×</button>
- <div>
- <div class="col-sm-6"> <h4 class="modal-title">Re-run container request</h4> </div>
- </div>
- <br/>
- </div>
-
- <div class="modal-body">
- <%= check_box_tag(:use_existing, "true", false) %>
- <%= label_tag(:use_existing, "Enable container reuse") %>
- </div>
-
- <div class="modal-footer">
- <button class="btn btn-default" onClick="reset_form_cr_reuse()" data-dismiss="modal" aria-hidden="true">Cancel</button>
- <button type="submit" class="btn btn-primary" name="container_request[state]" value="Uncommitted">Copy and edit inputs</button>
- </div>
-
- </div>
+ <%= link_to(choose_projects_path(id: "run-workflow-button",
+ title: 'Choose project',
+ editable: true,
+ action_name: 'Choose',
+ action_href: copy_container_request_path,
+ action_method: 'post',
+ action_data: {'selection_param' => 'work_unit[owner_uuid]',
+ 'work_unit[template_uuid]' => @object.uuid,
+ 'success' => 'redirect-to-created-object'
+ }.to_json),
+ { class: "btn btn-primary btn-sm", title: "Run #{@object.name}", remote: true }
+ ) do %>
+ <i class="fa fa-fw fa-play"></i> Re-run...
<% end %>
- </div>
-</div>
<% end %>
<% if workflow %>
<% inputs = get_cwl_inputs(workflow) %>
<% inputs.each do |input| %>
- <label for="#input-<%= cwl_shortname(input[:id]) %>">
- <%= input[:label] || cwl_shortname(input[:id]) %>
- </label>
- <div>
- <p class="form-control-static">
- <%= render_cwl_input @object, input, [:mounts, :"/var/lib/cwl/cwl.input.json", :content] %>
+ <div class="form-control-static">
+ <label for="#input-<%= cwl_shortname(input[:id]) %>">
+ <%= input[:label] || cwl_shortname(input[:id]) %>
+ </label>
+ <%= render_cwl_input @object, input, [:mounts, :"/var/lib/cwl/cwl.input.json", :content] %>
+ <p class="help-block">
+ <%= input[:doc] %>
</p>
</div>
- <p class="help-block">
- <%= input[:doc] %>
- </p>
<% end %>
<% end %>
</div>
</form>
<% end %>
+<p style="margin-bottom: 2em"><b style="margin-right: 3em">Reuse past workflow steps if available?</b> <%= render_editable_attribute(@object, :reuse_steps) %></p>
+
<% if n_inputs == 0 %>
<p><i>This workflow does not need any further inputs specified. Click the "Run" button at the bottom of the page to start the workflow.</i></p>
<% else %>
</div>
<% profile_config.kind_of?(Array) && profile_config.andand.each do |entry| %>
- <% if entry['Key'] %>
+ <% if entry[:Key] %>
<%
show_save_button = true
- label = entry['Required'] ? '* ' : ''
- label += entry['FormFieldTitle']
- value = current_user_profile[entry['Key'].to_sym] if current_user_profile
+ label = entry[:Required] ? '* ' : ''
+ label += entry[:FormFieldTitle]
+ value = current_user_profile[entry[:Key].to_sym] if current_user_profile
%>
<div class="form-group">
- <label for="<%=entry['Key']%>"
+ <label for="<%=entry[:Key]%>"
class="col-sm-3 control-label"
- style=<%="color:red" if entry['Required']&&(!value||value.empty?)%>> <%=label%>
+ style=<%="color:red" if entry[:Required]&&(!value||value.empty?)%>> <%=label%>
</label>
- <% if entry['Type'] == 'select' %>
+ <% if entry[:Type] == 'select' %>
<div class="col-sm-8">
- <select class="form-control" name="user[prefs][profile][<%=entry['Key']%>]">
- <% entry['Options'].each do |option, _| %>
+ <select class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]">
+ <% entry[:Options].each do |option, _| %>
+ <% option = option.to_s %>
<option value="<%=option%>" <%='selected' if option==value%>><%=option%></option>
<% end %>
</select>
</div>
<% else %>
<div class="col-sm-8">
- <input type="text" class="form-control" name="user[prefs][profile][<%=entry['Key']%>]" placeholder="<%=entry['FormFieldDescription']%>" value="<%=value%>" ></input>
+ <input type="text" class="form-control" name="user[prefs][profile][<%=entry[:Key]%>]" placeholder="<%=entry[:FormFieldDescription]%>" value="<%=value%>" ></input>
</div>
<% end %>
</div>
get :show, params: {id: uuid}, session: session_for(:active)
assert_response :success
- assert_includes @response.body, "action=\"/container_requests/#{uuid}/copy\""
+ assert_includes @response.body, "action_href=%2Fcontainer_requests%2F#{uuid}%2Fcopy"
end
test "cancel request for queued container" do
end
[
- ['completed', false, false],
- ['completed', true, false],
+ ['completed', false, false],
+ ['completed', true, false],
+ ['completed', nil, false],
['completed-older', false, true],
- ['completed-older', true, true],
+ ['completed-older', true, true],
+ ['completed-older', nil, true],
].each do |cr_fixture, reuse_enabled, uses_acr|
- test "container request #{uses_acr ? '' : 'not'} using arvados-cwl-runner copy #{reuse_enabled ? 'with' : 'without'} reuse enabled" do
+ test "container request #{uses_acr ? '' : 'not'} using arvados-cwl-runner copy #{reuse_enabled.nil? ? 'nil' : (reuse_enabled ? 'with' : 'without')} reuse enabled" do
completed_cr = api_fixture('container_requests')[cr_fixture]
# Set up post request params
copy_params = {id: completed_cr['uuid']}
- if reuse_enabled
- copy_params.merge!({use_existing: true})
+ if !reuse_enabled.nil?
+ copy_params.merge!({use_existing: reuse_enabled})
end
post(:copy, params: copy_params, session: session_for(:active))
assert_response 302
# If the CR's command is arvados-cwl-runner, the appropriate flag should
# be passed to it
if uses_acr
- if reuse_enabled
- # arvados-cwl-runner's default behavior is to enable reuse
- assert_includes copied_cr['command'], 'arvados-cwl-runner'
+ assert_equal copied_cr['command'][0], 'arvados-cwl-runner'
+ if reuse_enabled.nil? || reuse_enabled
+ assert_includes copied_cr['command'], '--enable-reuse'
assert_not_includes copied_cr['command'], '--disable-reuse'
else
- assert_includes copied_cr['command'], 'arvados-cwl-runner'
assert_includes copied_cr['command'], '--disable-reuse'
assert_not_includes copied_cr['command'], '--enable-reuse'
end
assert_text process_txt
assert_selector 'a', text: template_name
- assert_equal "Set value for ex_string_def", find('div.form-group > div > p.form-control-static > a', text: "hello-testing-123")[:"data-title"]
+ assert_equal "true", find('span[data-name="reuse_steps"]').text
+
+ assert_equal "Set value for ex_string_def", find('div.form-group > div.form-control-static > a', text: "hello-testing-123")[:"data-title"]
page.assert_selector 'a.disabled,button.disabled', text: 'Run'
end
inside Docker container with proper
build environment.
-run-build-packages-sso.sh Build single-sign-on server packages.
-
run-build-packages-python-and-ruby.sh Build Python and Ruby packages suitable
for upload to PyPi and Rubygems.
run-library.sh A library of functions shared by the
various scripts in this
- directory.
\ No newline at end of file
+ directory.
WORKSPACE=path Path to the Arvados source tree to build packages from
CWLTOOL=path (optional) Path to cwltool git repository.
SALAD=path (optional) Path to schema_salad git repository.
-PYCMD=pythonexec (optional) Specify the python executable to use in the docker image. Defaults to "python3".
+PYCMD=pythonexec (optional) Specify the python3 executable to use in the docker image. Defaults to "python3".
EOF
pipcmd=pip3
fi
-(cd sdk/python && python setup.py sdist)
+(cd sdk/python && python3 setup.py sdist)
sdk=$(cd sdk/python/dist && ls -t arvados-python-client-*.tar.gz | head -n1)
-(cd sdk/cwl && python setup.py sdist)
+(cd sdk/cwl && python3 setup.py sdist)
runner=$(cd sdk/cwl/dist && ls -t arvados-cwl-runner-*.tar.gz | head -n1)
rm -rf sdk/cwl/salad_dist
mkdir -p sdk/cwl/salad_dist
if [[ -n "$SALAD" ]] ; then
- (cd "$SALAD" && python setup.py sdist)
+ (cd "$SALAD" && python3 setup.py sdist)
salad=$(cd "$SALAD/dist" && ls -t schema-salad-*.tar.gz | head -n1)
cp "$SALAD/dist/$salad" $WORKSPACE/sdk/cwl/salad_dist
fi
rm -rf sdk/cwl/cwltool_dist
mkdir -p sdk/cwl/cwltool_dist
if [[ -n "$CWLTOOL" ]] ; then
- (cd "$CWLTOOL" && python setup.py sdist)
+ (cd "$CWLTOOL" && python3 setup.py sdist)
cwltool=$(cd "$CWLTOOL/dist" && ls -t cwltool-*.tar.gz | head -n1)
cp "$CWLTOOL/dist/$cwltool" $WORKSPACE/sdk/cwl/cwltool_dist
fi
. build/run-library.sh
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
set -x
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian9/generated debian10/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian10/generated ubuntu1604/generated ubuntu1804/generated ubuntu2004/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
- cp -rlt centos7/generated common-generated/*
-
-debian9/generated: common-generated-all
- test -d debian9/generated || mkdir debian9/generated
- cp -rlt debian9/generated common-generated/*
+ cp -f -rlt centos7/generated common-generated/*
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
- cp -rlt debian10/generated common-generated/*
-
+ cp -f -rlt debian10/generated common-generated/*
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
- cp -rlt ubuntu1604/generated common-generated/*
+ cp -f -rlt ubuntu1604/generated common-generated/*
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
- cp -rlt ubuntu1804/generated common-generated/*
+ cp -f -rlt ubuntu1804/generated common-generated/*
+
+ubuntu2004/generated: common-generated-all
+ test -d ubuntu2004/generated || mkdir ubuntu2004/generated
+ cp -f -rlt ubuntu2004/generated common-generated/*
GOTARBALL=go1.13.4.linux-amd64.tar.gz
NODETARBALL=node-v6.11.2-linux-x64.tar.xz
# Need to "touch" RPM database to workaround bug in interaction between
# overlayfs and yum (https://bugzilla.redhat.com/show_bug.cgi?id=1213602)
-RUN touch /var/lib/rpm/* && yum -q -y install rh-python36
-RUN scl enable rh-python36 "easy_install-3.6 pip"
+RUN touch /var/lib/rpm/* && yum -q -y install python3 python3-pip python3-devel
-# Add epel, we need it for the python-pam dependency
-#RUN wget http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-#RUN rpm -ivh epel-release-latest-7.noarch.rpm
+# Install virtualenv
+RUN /usr/bin/pip3 install 'virtualenv<20'
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
# The version of setuptools that comes with CentOS is way too old
-RUN scl enable rh-python36 "easy_install-3.6 pip install 'setuptools<45'"
+RUN pip3 install 'setuptools<45'
ENV WORKSPACE /arvados
-CMD ["scl", "enable", "rh-python36", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "centos7"]
#
# SPDX-License-Identifier: AGPL-3.0
-## dont use debian:9 here since the word 'stretch' is used for rvm precompiled binaries
-FROM debian:stretch
+FROM ubuntu:focal
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies.
-RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-dev libpam-dev
+RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev
# Install virtualenv
RUN /usr/bin/pip3 install 'virtualenv<20'
RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle
ENV WORKSPACE /arvados
-CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
+CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2004"]
#
# SPDX-License-Identifier: AGPL-3.0
-all: centos7/generated debian9/generated debian10/generated ubuntu1604/generated ubuntu1804/generated
+all: centos7/generated debian10/generated ubuntu1604/generated ubuntu1804/generated ubuntu2004/generated
centos7/generated: common-generated-all
test -d centos7/generated || mkdir centos7/generated
- cp -rlt centos7/generated common-generated/*
-
-debian9/generated: common-generated-all
- test -d debian9/generated || mkdir debian9/generated
- cp -rlt debian9/generated common-generated/*
+ cp -f -rlt centos7/generated common-generated/*
debian10/generated: common-generated-all
test -d debian10/generated || mkdir debian10/generated
- cp -rlt debian10/generated common-generated/*
+ cp -f -rlt debian10/generated common-generated/*
ubuntu1604/generated: common-generated-all
test -d ubuntu1604/generated || mkdir ubuntu1604/generated
- cp -rlt ubuntu1604/generated common-generated/*
+ cp -f -rlt ubuntu1604/generated common-generated/*
ubuntu1804/generated: common-generated-all
test -d ubuntu1804/generated || mkdir ubuntu1804/generated
- cp -rlt ubuntu1804/generated common-generated/*
+ cp -f -rlt ubuntu1804/generated common-generated/*
+
+ubuntu2004/generated: common-generated-all
+ test -d ubuntu2004/generated || mkdir ubuntu2004/generated
+ cp -f -rlt ubuntu2004/generated common-generated/*
RVMKEY1=mpapis.asc
RVMKEY2=pkuczynski.asc
#
# SPDX-License-Identifier: AGPL-3.0
-FROM debian:stretch
+FROM ubuntu:focal
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
# Install dependencies
RUN apt-get update && \
- apt-get -y install --no-install-recommends curl ca-certificates gpg procps
+ apt-get -y install --no-install-recommends curl ca-certificates gnupg2
# Install RVM
ADD generated/mpapis.asc /tmp/
# udev daemon can't start in a container, so don't try.
RUN mkdir -p /etc/udev/disabled
-RUN echo "deb file:///arvados/packages/debian9/ /" >>/etc/apt/sources.list
+RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu2004/ /" >>/etc/apt/sources.list
arv-put --version
-/usr/share/python3/dist/rh-python36-python-arvados-python-client/bin/python3 << EOF
+/usr/bin/python3 << EOF
import arvados
print("Successfully imported arvados")
EOF
--- /dev/null
+deb-common-test-packages.sh
\ No newline at end of file
* After it installs the core configuration files (database.yml, application.yml, and production.rb) to /etc/arvados/server, it calls setup_extra_conffiles. By default this is a noop function (in step2.sh).
* Before it restarts nginx, it calls setup_before_nginx_restart. By default this is a noop function (in step2.sh). API server defines this to set up the internal git repository, if necessary.
-* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. SSO server uses db:schema:load. Workbench doesn't set this, which causes the postinst to skip all database work.
-* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task. SSO clears this flag (it doesn't have that task code).
+* $RAILSPKG_DATABASE_LOAD_TASK defines the Rake task to load the database. API server uses db:structure:load. Workbench doesn't set this, which causes the postinst to skip all database work.
+* If $RAILSPKG_SUPPORTS_CONFIG_CHECK != 1, it won't run the config:check rake task.
+++ /dev/null
-#!/bin/sh
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This file declares variables common to all scripts for one Rails package.
-
-PACKAGE_NAME=arvados-sso-server
-INSTALL_PATH=/var/www/arvados-sso
-CONFIG_PATH=/etc/arvados/sso
-DOC_URL="https://doc.arvados.org/v2.0/install/install-sso.html#configure"
-RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
-RAILSPKG_SUPPORTS_CONFIG_CHECK=0
chown "$WWW_OWNER:" $RELEASE_PATH/Gemfile.lock
chown -R "$WWW_OWNER:" $RELEASE_PATH/tmp || true
chown -R "$WWW_OWNER:" $SHARED_PATH/log
+ # Make sure postgres doesn't try to use a pager.
+ export PAGER=
case "$RAILSPKG_DATABASE_LOAD_TASK" in
db:schema:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/schema.rb ;;
db:structure:load) chown "$WWW_OWNER:" $RELEASE_PATH/db/structure.sql ;;
configure_version
fi
-if printf '%s\n' "$CONFIG_PATH" | grep -Fqe "sso"; then
- report_not_ready "$APPLICATION_READY" "$CONFIG_PATH/application.yml"
- report_not_ready "$DATABASE_READY" "$CONFIG_PATH/database.yml"
-else
- report_not_ready "$APPLICATION_READY" "/etc/arvados/config.yml"
-fi
+report_not_ready "$APPLICATION_READY" "/etc/arvados/config.yml"
echo >&2
echo >&2 "$0 options:"
echo >&2 " -t, --tags [csv_tags] comma separated tags"
+ echo >&2 " -i, --images [dev,demo] Choose which images to build (default: dev and demo)"
echo >&2 " -u, --upload Upload the images (docker push)"
echo >&2 " -h, --help Display this help and exit"
echo >&2
}
upload=false
+images=dev,demo
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hut: \
- --long help,upload,tags: \
+TEMP=`getopt -o hut:i: \
+ --long help,upload,tags:,images: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
upload=true
shift
;;
+ -i | --images)
+ case "$2" in
+ "")
+ echo "ERROR: --images needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ images=$2;
+ shift 2
+ ;;
+ esac
+ ;;
-t | --tags)
case "$2" in
"")
}
docker_push () {
+ # docker always creates a local 'latest' tag, and we don't want to push that
+ # tag in every case. Remove it.
+ docker rmi $1:latest
if [[ ! -z "$tags" ]]
then
for tag in $( echo $tags|tr "," " " )
# clean up the docker build environment
cd "$WORKSPACE"
-title "Starting arvbox build localdemo"
+if [[ "$images" =~ demo ]]; then
+ title "Starting arvbox build localdemo"
-tools/arvbox/bin/arvbox build localdemo
-ECODE=$?
+ tools/arvbox/bin/arvbox build localdemo
+ ECODE=$?
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker BUILD FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
fi
-title "Starting arvbox build dev"
+if [[ "$images" =~ dev ]]; then
+ title "Starting arvbox build dev"
-tools/arvbox/bin/arvbox build dev
+ tools/arvbox/bin/arvbox build dev
-ECODE=$?
+ ECODE=$?
-if [[ "$ECODE" != "0" ]]; then
- title "!!!!!! docker BUILD FAILED !!!!!!"
- EXITCODE=$(($EXITCODE + $ECODE))
+ if [[ "$ECODE" != "0" ]]; then
+ title "!!!!!! docker BUILD FAILED !!!!!!"
+ EXITCODE=$(($EXITCODE + $ECODE))
+ fi
fi
title "docker build complete (`timer`)"
-title "uploading images"
-
-timer_reset
-
if [[ "$EXITCODE" != "0" ]]; then
title "upload arvados images SKIPPED because build failed"
else
if [[ $upload == true ]]; then
+ title "uploading images"
+ timer_reset
+
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- docker_push arvados/arvbox-dev
- docker_push arvados/arvbox-demo
+ if [[ "$images" =~ dev ]]; then
+ docker_push arvados/arvbox-dev
+ fi
+ if [[ "$images" =~ demo ]]; then
+ docker_push arvados/arvbox-demo
+ fi
title "upload arvados images complete (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set"
ARVADOS_BUILDING_ITERATION="1"
fi
+# This defines python_sdk_version and cwl_runner_version with python-style
+# package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
+if [[ -z "$cwl_runner_version" ]]; then
+ echo "ERROR: cwl_runner_version is empty";
+ exit 1
+fi
+
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
+# For development and release candidate packages, the OS package has a "~dev"
+# or "~rc" suffix, but Python requires a ".dev" or "rc" suffix.
+#
+# Arvados-cwl-runner will be expecting the Python-compatible version string
+# when it tries to pull the Docker image, so we use that to tag the Docker
+# image.
+#
+# The --build-arg docker invocation arguments are expecting the OS package
+# version.
+python_sdk_version_os=$(echo -n $python_sdk_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
+cwl_runner_version_os=$(echo -n $cwl_runner_version | sed s/.dev/~dev/g | sed s/rc/~rc/g)
+
if [[ "${python_sdk_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- python_sdk_version="${python_sdk_version}-1"
+ python_sdk_version_os="${python_sdk_version_os}-1"
else
- python_sdk_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+ python_sdk_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
-cwl_runner_version_orig=$cwl_runner_version
-
-if [[ "${cwl_runner_version}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
- cwl_runner_version="${cwl_runner_version}-1"
+if [[ "${cwl_runner_version_os}" != "${ARVADOS_BUILDING_VERSION}" ]]; then
+ cwl_runner_version_os="${cwl_runner_version_os}-1"
else
- cwl_runner_version="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
+ cwl_runner_version_os="${ARVADOS_BUILDING_VERSION}-${ARVADOS_BUILDING_ITERATION}"
fi
cd docker/jobs
docker build $NOCACHE \
- --build-arg python_sdk_version=${python_sdk_version} \
- --build-arg cwl_runner_version=${cwl_runner_version} \
+ --build-arg python_sdk_version=${python_sdk_version_os} \
+ --build-arg cwl_runner_version=${cwl_runner_version_os} \
--build-arg repo_version=${REPO} \
- -t arvados/jobs:$cwl_runner_version_orig .
+ -t arvados/jobs:$cwl_runner_version .
ECODE=$?
FORCE=-f
fi
-if ! [[ -z "$version_tag" ]]; then
- docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
- ECODE=$?
-
- if [[ "$ECODE" != "0" ]]; then
- EXITCODE=$(($EXITCODE + $ECODE))
- fi
-
- checkexit $ECODE "docker tag"
- title "docker tag complete (`timer`)"
-fi
-
title "uploading images"
timer_reset
## 20150526 nico -- *sometimes* dockerhub needs re-login
## even though credentials are already in .dockercfg
docker login -u arvados
- if ! [[ -z "$version_tag" ]]; then
- docker_push arvados/jobs:"$version_tag"
- else
- docker_push arvados/jobs:$cwl_runner_version_orig
- fi
+ docker_push arvados/jobs:$cwl_runner_version
title "upload arvados images finished (`timer`)"
else
title "upload arvados images SKIPPED because no --upload option set (`timer`)"
keep-block-check
keep-web
libarvados-perl
- libpam-arvados-go"
- if [[ "$TARGET" =~ "centos" ]]; then
- packages="$packages
- rh-python36-python-cwltest
- rh-python36-python-arvados-fuse
- rh-python36-python-arvados-python-client
- rh-python36-python-arvados-cwl-runner
- rh-python36-python-crunchstat-summary"
- else
- packages="$packages
+ libpam-arvados-go
python3-cwltest
python3-arvados-fuse
python3-arvados-python-client
python3-arvados-cwl-runner
- python3-crunchstat-summary"
- fi
+ python3-crunchstat-summary
+ python3-arvados-user-activity"
fi
FINAL_EXITCODE=0
COLUMNS=80
. `dirname "$(readlink -f "$0")"`/run-library.sh
-#. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
read -rd "\000" helpmessage <<EOF
$(basename $0): Build Arvados Python packages and Ruby gems
title "End of $gem_name gem build (`timer`)"
}
+handle_python_package () {
+ # This function assumes the current working directory is the python package directory
+ if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
+ echo "This package doesn't need rebuilding."
+ return
+ fi
+ # Make sure only to use sdist - that's the only format pip can deal with (sigh)
+ python3 setup.py $DASHQ_UNLESS_DEBUG sdist
+}
+
python_wrapper() {
local package_name="$1"; shift
local package_directory="$1"; shift
python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+ python_wrapper crunchstat_summary "$WORKSPACE/tools/crunchstat-summary"
+ python_wrapper arvados-user-activity "$WORKSPACE/tools/user-activity"
if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
+++ /dev/null
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-JENKINS_DIR=$(dirname $(readlink -e "$0"))
-. "$JENKINS_DIR/run-library.sh"
-
-read -rd "\000" helpmessage <<EOF
-$(basename $0): Build Arvados SSO server package
-
-Syntax:
- WORKSPACE=/path/to/arvados-sso $(basename $0) [options]
-
-Options:
-
---debug
- Output debug information (default: false)
---target
- Distribution to build packages for (default: debian10)
-
-WORKSPACE=path Path to the Arvados SSO source tree to build packages from
-
-EOF
-
-EXITCODE=0
-DEBUG=${ARVADOS_DEBUG:-0}
-TARGET=debian10
-
-PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,build-bundle-packages,debug,target: \
- -- "" "$@")
-if [ $? -ne 0 ]; then
- exit 1
-fi
-
-eval set -- "$PARSEDOPTS"
-while [ $# -gt 0 ]; do
- case "$1" in
- --help)
- echo >&2 "$helpmessage"
- echo >&2
- exit 1
- ;;
- --target)
- TARGET="$2"; shift
- ;;
- --debug)
- DEBUG=1
- ;;
- --test-packages)
- test_packages=1
- ;;
- --)
- if [ $# -gt 1 ]; then
- echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
- exit 1
- fi
- ;;
- esac
- shift
-done
-
-STDOUT_IF_DEBUG=/dev/null
-STDERR_IF_DEBUG=/dev/null
-DASHQ_UNLESS_DEBUG=-q
-if [[ "$DEBUG" != 0 ]]; then
- STDOUT_IF_DEBUG=/dev/stdout
- STDERR_IF_DEBUG=/dev/stderr
- DASHQ_UNLESS_DEBUG=
-fi
-
-case "$TARGET" in
- debian*)
- FORMAT=deb
- ;;
- ubuntu*)
- FORMAT=deb
- ;;
- centos*)
- FORMAT=rpm
- ;;
- *)
- echo -e "$0: Unknown target '$TARGET'.\n" >&2
- exit 1
- ;;
-esac
-
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-if ! [[ -d "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: $WORKSPACE is not a directory"
- echo >&2
- exit 1
-fi
-
-# Test for fpm
-fpm --version >/dev/null 2>&1
-
-if [[ "$?" != 0 ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: fpm not found"
- echo >&2
- exit 1
-fi
-
-RUN_BUILD_PACKAGES_PATH="`dirname \"$0\"`"
-RUN_BUILD_PACKAGES_PATH="`( cd \"$RUN_BUILD_PACKAGES_PATH\" && pwd )`" # absolutized and normalized
-if [ -z "$RUN_BUILD_PACKAGES_PATH" ] ; then
- # error; for some reason, the path is not accessible
- # to the script (e.g. permissions re-evaled after suid)
- exit 1 # fail
-fi
-
-debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
-debug_echo "Workspace is $WORKSPACE"
-
-if [[ -f /etc/profile.d/rvm.sh ]]; then
- source /etc/profile.d/rvm.sh
- GEM="rvm-exec default gem"
-else
- GEM=gem
-fi
-
-# Make all files world-readable -- jenkins runs with umask 027, and has checked
-# out our git tree here
-chmod o+r "$WORKSPACE" -R
-
-# More cleanup - make sure all executables that we'll package are 755
-# No executables in the sso server package
-#find -type d -name 'bin' |xargs -I {} find {} -type f |xargs -I {} chmod 755 {}
-
-# Now fix our umask to something better suited to building and publishing
-# gems and packages
-umask 0022
-
-debug_echo "umask is" `umask`
-
-if [[ ! -d "$WORKSPACE/packages/$TARGET" ]]; then
- mkdir -p "$WORKSPACE/packages/$TARGET"
-fi
-
-# Build the SSO server package
-handle_rails_package arvados-sso-server "$WORKSPACE" \
- "$WORKSPACE/LICENCE" --url="https://arvados.org" \
- --description="Arvados SSO server - Arvados is a free and open source platform for big data science." \
- --license="Expat license"
-
-exit $EXITCODE
FORMAT=rpm
PYTHON3_PACKAGE=$(rpm -qf "$(which python$PYTHON3_VERSION)" --queryformat '%{NAME}\n')
PYTHON3_PKG_PREFIX=$PYTHON3_PACKAGE
- PYTHON3_PREFIX=/opt/rh/rh-python36/root/usr
+ PYTHON3_PREFIX=/usr
PYTHON3_INSTALL_LIB=lib/python$PYTHON3_VERSION/site-packages
export PYCURL_SSL_LIBRARY=nss
;;
git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
echo "$COMMIT_HASH" >git-commit.version
- cd "$SRC_BUILD_DIR"
- PKG_VERSION=$(version_from_git)
cd $WORKSPACE/packages/$TARGET
- fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+ fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
rm -rf "$SRC_BUILD_DIR"
fi
# The Docker image cleaner
fpm_build_virtualenv "arvados-docker-cleaner" "services/dockercleaner" "python3"
+# The Arvados user activity tool
+fpm_build_virtualenv "arvados-user-activity" "tools/user-activity" "python3"
+
# The cwltest package, which lives out of tree
cd "$WORKSPACE"
if [[ -e "$WORKSPACE/cwltest" ]]; then
}
nohash_version_from_git() {
+ local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git | cut -d. -f1-4
+ version_from_git $subdir | cut -d. -f1-4
}
timestamp_from_git() {
}
calculate_python_sdk_cwl_package_versions() {
- python_sdk_ts=$(cd sdk/python && timestamp_from_git)
- cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
- python_sdk_version=$(cd sdk/python && nohash_version_from_git)
- cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
-
- if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$python_sdk_version
- fi
-}
-
-handle_python_package () {
- # This function assumes the current working directory is the python package directory
- if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
- # This package doesn't need rebuilding.
- return
- fi
- # Make sure only to use sdist - that's the only format pip can deal with (sigh)
- python setup.py $DASHQ_UNLESS_DEBUG sdist
+ python_sdk_version=$(cd sdk/python && python3 arvados_version.py)
+ cwl_runner_version=$(cd sdk/cwl && python3 arvados_version.py)
}
handle_ruby_gem() {
checkdirs+=("$1")
shift
done
- if grep -qr git.arvados.org/arvados .; then
- checkdirs+=(sdk/go lib)
- fi
+ # Even our rails packages (version calculation happens here!) depend on a go component (arvados-server)
+ # Everything depends on the build directory.
+ checkdirs+=(sdk/go lib build)
local timestamp=0
for dir in ${checkdirs[@]}; do
cd "$WORKSPACE"
fi
local version="$(version_from_git)"
if [ $pkgname = "arvados-api-server" -o $pkgname = "arvados-workbench" ] ; then
- calculate_go_package_version version cmd/arvados-server "$srcdir"
+ calculate_go_package_version version cmd/arvados-server "$srcdir"
fi
echo $version
}
echo "Package $full_pkgname build forced with --force-build, building"
elif [[ "$FORMAT" == "deb" ]]; then
declare -A dd
- dd[debian9]=stretch
dd[debian10]=buster
dd[ubuntu1604]=xenial
dd[ubuntu1804]=bionic
+ dd[ubuntu2004]=focal
D=${dd[$TARGET]}
if [ ${pkgname:0:3} = "lib" ]; then
repo_subdir=${pkgname:0:4}
repo_subdir=${pkgname:0:1}
fi
- repo_pkg_list=$(curl -s -o - http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/)
+ repo_pkg_list=$(curl -s -o - http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/)
echo "${repo_pkg_list}" |grep -q ${full_pkgname}
if [ $? -eq 0 ] ; then
echo "Package $full_pkgname exists upstream, not rebuilding, downloading instead!"
- curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/pool/${D}-dev/main/${repo_subdir}/${pkgname}/${full_pkgname}
+ curl -s -o "$WORKSPACE/packages/$TARGET/${full_pkgname}" http://apt.arvados.org/${D}/pool/main/${repo_subdir}/${pkgname}/${full_pkgname}
return 1
elif test -f "$WORKSPACE/packages/$TARGET/processed/${full_pkgname}" ; then
echo "Package $full_pkgname exists, not rebuilding!"
fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
- # .git and packages are for the SSO server, which is built from its
- # repository root.
- local -a exclude_list=(.git packages tmp log coverage Capfile\* \
+ local -a exclude_list=(tmp log coverage Capfile\* \
config/deploy\* config/application.yml)
# for arvados-workbench, we need to have the (dummy) config/database.yml in the package
if [[ "$pkgname" != "arvados-workbench" ]]; then
case "$PACKAGE_TYPE" in
python3)
python=python3
- if [[ "$FORMAT" != "rpm" ]]; then
- pip=pip3
- else
- # In CentOS, we use a different mechanism to get the right version of pip
- pip=pip
- fi
+ pip=pip3
PACKAGE_PREFIX=$PYTHON3_PKG_PREFIX
;;
esac
fi
# Determine the package version from the generated sdist archive
- PYTHON_VERSION=${ARVADOS_BUILDING_VERSION:-$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)}
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]] ; then
+ UNFILTERED_PYTHON_VERSION=$ARVADOS_BUILDING_VERSION
+ PYTHON_VERSION=$(echo -n $ARVADOS_BUILDING_VERSION | sed s/~dev/.dev/g | sed s/~rc/rc/g)
+ else
+ PYTHON_VERSION=$(awk '($1 == "Version:"){print $2}' *.egg-info/PKG-INFO)
+ UNFILTERED_PYTHON_VERSION=$(echo -n $PYTHON_VERSION | sed s/\.dev/~dev/g |sed 's/\([0-9]\)rc/\1~rc/g')
+ fi
# See if we actually need to build this package; does it exist already?
# We can't do this earlier than here, because we need PYTHON_VERSION...
# This isn't so bad; the sdist call above is pretty quick compared to
# the invocation of virtualenv and fpm, below.
- if ! test_package_presence "$PYTHON_PKG" $PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
+ if ! test_package_presence "$PYTHON_PKG" $UNFILTERED_PYTHON_VERSION $PACKAGE_TYPE $ARVADOS_BUILDING_ITERATION; then
return 0
fi
COMMAND_ARR+=('--verbose' '--log' 'info')
fi
- COMMAND_ARR+=('-v' "$PYTHON_VERSION")
+ COMMAND_ARR+=('-v' $(echo -n "$PYTHON_VERSION" | sed s/.dev/~dev/g | sed s/rc/~rc/g))
COMMAND_ARR+=('--iteration' "$ARVADOS_BUILDING_ITERATION")
COMMAND_ARR+=('-n' "$PYTHON_PKG")
COMMAND_ARR+=('-C' "build")
done
fi
- # the python-arvados-cwl-runner package comes with cwltool, expose that version
- if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
- COMMAND_ARR+=("usr/share/python2.7/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
+ # the python3-arvados-cwl-runner package comes with cwltool, expose that version
+ if [[ -e "$WORKSPACE/$PKG_DIR/dist/build/usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool" ]]; then
+ COMMAND_ARR+=("usr/share/$python/dist/python-arvados-cwl-runner/bin/cwltool=/usr/bin/")
fi
COMMAND_ARR+=(".")
#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
set -e -o pipefail
commit="$1"
versionglob="[0-9].[0-9]*.[0-9]*"
-devsuffix=".dev"
+devsuffix="~dev"
# automatically assign version
#
"git.arvados.org/arvados.git/lib/cli"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/costanalyzer"
"git.arvados.org/arvados.git/lib/deduplicationreport"
"git.arvados.org/arvados.git/lib/mount"
)
"mount": mount.Command,
"deduplication-report": deduplicationreport.Command,
+ "costanalyzer": costanalyzer.Command,
})
)
source 'https://rubygems.org'
gem 'zenweb'
-gem 'liquid'
+gem 'liquid', '~>4.0.0'
gem 'RedCloth'
gem 'colorize'
DEPENDENCIES
RedCloth
colorize
- liquid
+ liquid (~> 4.0.0)
zenweb
BUNDLED WITH
- api/methods/virtual_machines.html.textile.liquid
- api/methods/keep_disks.html.textile.liquid
- Data management:
+ - api/keep-webdav.html.textile.liquid
+ - api/keep-s3.html.textile.liquid
+ - api/keep-web-urls.html.textile.liquid
- api/methods/collections.html.textile.liquid
- api/methods/repositories.html.textile.liquid
- Container engine:
- user/topics/arvados-sync-groups.html.textile.liquid
- admin/scoped-tokens.html.textile.liquid
- admin/token-expiration-policy.html.textile.liquid
+ - admin/user-activity.html.textile.liquid
- Monitoring:
- admin/logging.html.textile.liquid
- admin/metrics.html.textile.liquid
- install/index.html.textile.liquid
- Docker quick start:
- install/arvbox.html.textile.liquid
+ - Installation with Salt:
+ - install/salt.html.textile.liquid
+ - install/salt-vagrant.html.textile.liquid
+ - install/salt-single-host.html.textile.liquid
+ - install/salt-multi-host.html.textile.liquid
- Arvados on Kubernetes:
- install/arvados-on-kubernetes.html.textile.liquid
- install/arvados-on-kubernetes-minikube.html.textile.liquid
+++ /dev/null
-#!/usr/bin/env ruby
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-require 'rubygems'
-
-require 'cgi'
-require 'fileutils'
-require 'json'
-require 'net/https'
-require 'socket'
-require 'syslog'
-
-class ComputeNodePing
- @@NODEDATA_DIR = "/var/tmp/arv-node-data"
- @@PUPPET_CONFFILE = "/etc/puppet/puppet.conf"
- @@HOST_STATEFILE = "/var/run/arvados-compute-ping-hoststate.json"
-
- def initialize(args, stdout, stderr)
- @stdout = stdout
- @stderr = stderr
- @stderr_loglevel = ((args.first == "quiet") ?
- Syslog::LOG_ERR : Syslog::LOG_DEBUG)
- @puppet_disabled = false
- @syslog = Syslog.open("arvados-compute-ping",
- Syslog::LOG_CONS | Syslog::LOG_PID,
- Syslog::LOG_DAEMON)
- @puppetless = File.exist?('/compute-node.puppetless')
-
- begin
- prepare_ping
- load_puppet_conf unless @puppetless
- begin
- @host_state = JSON.parse(IO.read(@@HOST_STATEFILE))
- rescue Errno::ENOENT
- @host_state = nil
- end
- rescue
- @syslog.close
- raise
- end
- end
-
- def send
- pong = send_raw_ping
-
- if pong["hostname"] and pong["domain"] and pong["first_ping_at"]
- if @host_state.nil?
- @host_state = {
- "fqdn" => (Socket.gethostbyname(Socket.gethostname).first rescue nil),
- "resumed_slurm" =>
- ["busy", "idle"].include?(pong["crunch_worker_state"]),
- }
- update_host_state({})
- end
-
- if hostname_changed?(pong)
- disable_puppet unless @puppetless
- rename_host(pong)
- update_host_state("fqdn" => fqdn_from_pong(pong),
- "resumed_slurm" => false)
- end
-
- unless @host_state["resumed_slurm"]
- run_puppet_agent unless @puppetless
- resume_slurm_node(pong["hostname"])
- update_host_state("resumed_slurm" => true)
- end
- end
-
- log("Last ping at #{pong['last_ping_at']}")
- end
-
- def cleanup
- enable_puppet if @puppet_disabled and not @puppetless
- @syslog.close
- end
-
- private
-
- def log(message, level=Syslog::LOG_INFO)
- @syslog.log(level, message)
- if level <= @stderr_loglevel
- @stderr.write("#{Time.now.strftime("%Y-%m-%d %H:%M:%S")} #{message}\n")
- end
- end
-
- def abort(message, code=1)
- log(message, Syslog::LOG_ERR)
- exit(code)
- end
-
- def run_and_check(cmd_a, accept_codes, io_opts, &block)
- result = IO.popen(cmd_a, "r", io_opts, &block)
- unless accept_codes.include?($?.exitstatus)
- abort("#{cmd_a} exited #{$?.exitstatus}")
- end
- result
- end
-
- DEFAULT_ACCEPT_CODES=[0]
- def check_output(cmd_a, accept_codes=DEFAULT_ACCEPT_CODES, io_opts={})
- # Run a command, check the exit status, and return its stdout as a string.
- run_and_check(cmd_a, accept_codes, io_opts) do |pipe|
- pipe.read
- end
- end
-
- def check_command(cmd_a, accept_codes=DEFAULT_ACCEPT_CODES, io_opts={})
- # Run a command, send stdout to syslog, and check the exit status.
- run_and_check(cmd_a, accept_codes, io_opts) do |pipe|
- pipe.each_line do |line|
- line.chomp!
- log("#{cmd_a.first}: #{line}") unless line.empty?
- end
- end
- end
-
- def replace_file(path, body)
- open(path, "w") { |f| f.write(body) }
- end
-
- def update_host_state(updates_h)
- @host_state.merge!(updates_h)
- replace_file(@@HOST_STATEFILE, @host_state.to_json)
- end
-
- def disable_puppet
- check_command(["puppet", "agent", "--disable"])
- @puppet_disabled = true
- loop do
- # Wait for any running puppet agents to finish.
- check_output(["pgrep", "puppet"], 0..1)
- break if $?.exitstatus == 1
- sleep(1)
- end
- end
-
- def enable_puppet
- check_command(["puppet", "agent", "--enable"])
- @puppet_disabled = false
- end
-
- def prepare_ping
- begin
- ping_uri_s = File.read(File.join(@@NODEDATA_DIR, "arv-ping-url"))
- rescue Errno::ENOENT
- abort("ping URL file is not present yet, skipping run")
- end
-
- ping_uri = URI.parse(ping_uri_s)
- payload_h = CGI.parse(ping_uri.query)
-
- # Collect all extra data to be sent
- dirname = File.join(@@NODEDATA_DIR, "meta-data")
- Dir.open(dirname).each do |basename|
- filename = File.join(dirname, basename)
- if File.file?(filename)
- payload_h[basename.gsub('-', '_')] = File.read(filename).chomp
- end
- end
-
- ping_uri.query = nil
- @ping_req = Net::HTTP::Post.new(ping_uri.to_s)
- @ping_req.set_form_data(payload_h)
- @ping_client = Net::HTTP.new(ping_uri.host, ping_uri.port)
- @ping_client.use_ssl = ping_uri.scheme == 'https'
- end
-
- def send_raw_ping
- begin
- response = @ping_client.start do |http|
- http.request(@ping_req)
- end
- if response.is_a? Net::HTTPSuccess
- pong = JSON.parse(response.body)
- else
- raise "response was a #{response}"
- end
- rescue JSON::ParserError => error
- abort("Error sending ping: could not parse JSON response: #{error}")
- rescue => error
- abort("Error sending ping: #{error}")
- end
-
- replace_file(File.join(@@NODEDATA_DIR, "pong.json"), response.body)
- if pong["errors"] then
- log(pong["errors"].join("; "), Syslog::LOG_ERR)
- if pong["errors"].grep(/Incorrect ping_secret/).any?
- system("halt")
- end
- exit(1)
- end
- pong
- end
-
- def load_puppet_conf
- # Parse Puppet configuration suitable for rewriting.
- # Save certnames in @puppet_certnames.
- # Save other functional configuration lines in @puppet_conf.
- @puppet_conf = []
- @puppet_certnames = []
- open(@@PUPPET_CONFFILE, "r") do |conffile|
- conffile.each_line do |line|
- key, value = line.strip.split(/\s*=\s*/, 2)
- if key == "certname"
- @puppet_certnames << value
- elsif not (key.nil? or key.empty? or key.start_with?("#"))
- @puppet_conf << line
- end
- end
- end
- end
-
- def fqdn_from_pong(pong)
- "#{pong['hostname']}.#{pong['domain']}"
- end
-
- def certname_from_pong(pong)
- fqdn = fqdn_from_pong(pong).sub(".", ".compute.")
- "#{pong['first_ping_at'].gsub(':', '-').downcase}.#{fqdn}"
- end
-
- def hostname_changed?(pong)
- if @puppetless
- (@host_state["fqdn"] != fqdn_from_pong(pong))
- else
- (@host_state["fqdn"] != fqdn_from_pong(pong)) or
- (@puppet_certnames != [certname_from_pong(pong)])
- end
- end
-
- def rename_host(pong)
- new_fqdn = fqdn_from_pong(pong)
- log("Renaming host from #{@host_state["fqdn"]} to #{new_fqdn}")
-
- replace_file("/etc/hostname", "#{new_fqdn.split('.', 2).first}\n")
- check_output(["hostname", new_fqdn])
-
- ip_address = check_output(["facter", "ipaddress"]).chomp
- esc_address = Regexp.escape(ip_address)
- check_command(["sed", "-i", "/etc/hosts",
- "-e", "s/^#{esc_address}.*$/#{ip_address}\t#{new_fqdn}/"])
-
- unless @puppetless
- new_conflines = @puppet_conf + ["\n[agent]\n",
- "certname=#{certname_from_pong(pong)}\n"]
- replace_file(@@PUPPET_CONFFILE, new_conflines.join(""))
- FileUtils.remove_entry_secure("/var/lib/puppet/ssl")
- end
- end
-
- def run_puppet_agent
- log("Running puppet agent")
- enable_puppet
- check_command(["puppet", "agent", "--onetime", "--no-daemonize",
- "--no-splay", "--detailed-exitcodes",
- "--ignorecache", "--no-usecacheonfailure"],
- [0, 2], {err: [:child, :out]})
- end
-
- def resume_slurm_node(node_name)
- current_state = check_output(["sinfo", "--noheader", "-o", "%t",
- "-n", node_name]).chomp
- if %w(down drain drng).include?(current_state)
- log("Resuming node in SLURM")
- check_command(["scontrol", "update", "NodeName=#{node_name}",
- "State=RESUME"], [0], {err: [:child, :out]})
- end
- end
-end
-
-LOCK_DIRNAME = "/var/lock/arvados-compute-node.lock"
-begin
- Dir.mkdir(LOCK_DIRNAME)
-rescue Errno::EEXIST
- exit(0)
-end
-
-ping_sender = nil
-begin
- ping_sender = ComputeNodePing.new(ARGV, $stdout, $stderr)
- ping_sender.send
-ensure
- Dir.rmdir(LOCK_DIRNAME)
- ping_sender.cleanup unless ping_sender.nil?
-end
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: CC-BY-SA-3.0
package main
h3. Debian and Ubuntu
-Debian 9 (stretch) and Ubuntu 16.04 (xenial) ship Ruby 2.3, which is not supported by Arvados. Use "RVM":#rvm to install Ruby 2.5 or later.
+Ubuntu 16.04 (xenial) ships with Ruby 2.3, which is not supported by Arvados. Use "RVM":#rvm to install Ruby 2.5 or later.
Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5, which is supported by Arvados.
h2(#fromsource). Option 3: Install from source
-Install prerequisites for Debian 8:
+Install prerequisites for Debian 10:
<notextile>
<pre><code><span class="userinput">sudo apt-get install \
- bison build-essential gettext libcurl3 libcurl3-gnutls \
+ bison build-essential gettext libcurl4 \
libcurl4-openssl-dev libpcre3-dev libreadline-dev \
libssl-dev libxslt1.1 zlib1g-dev
</span></code></pre></notextile>
make automake libtool bison sqlite-devel tar
</span></code></pre></notextile>
-Install prerequisites for Ubuntu 12.04 or 14.04:
+Install prerequisites for Ubuntu 16.04:
<notextile>
<pre><code><span class="userinput">sudo apt-get install \
- gawk g++ gcc make libc6-dev libreadline6-dev zlib1g-dev libssl-dev \
- libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev \
- libncurses5-dev automake libtool bison pkg-config libffi-dev curl
+ bison build-essential gettext libcurl3 \
+ libcurl3-openssl-dev libpcre3-dev libreadline-dev \
+ libssl-dev libxslt1.1 zlib1g-dev
</span></code></pre></notextile>
Build and install Ruby:
#!/usr/bin/env cwl-runner
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
cwlVersion: v1.0
class: CommandLineTool
inputs: []
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The master Arvados configuration is stored at @/etc/arvados/config.yml@
+The Arvados configuration is stored at @/etc/arvados/config.yml@
See "Migrating Configuration":config-migration.html for information about migrating from legacy component-specific configuration files.
LoginCluster: clsr1
</pre>
-The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the master cluster.
+The @LoginCluster@ configuration redirects all user logins to the LoginCluster, and the LoginCluster will issue API tokens which will be accepted by the federation. Users are activated or deactivated across the entire federation based on their status on the login cluster.
-Note: tokens issued by the master cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
+Note: tokens issued by the login cluster need to be periodically re-validated when used on other clusters in the federation. The period between revalidation attempts is configured with @Login.RemoteTokenRefresh@. The default is 5 minutes. A longer period reduces overhead from validating tokens, but means it may take longer for other clusters to notice when a token has been revoked or a user has changed status (being activated/deactivated, admin flag changed).
To migrate users of existing clusters with separate user databases to use a single LoginCluster, use "arv-federation-migrate":merge-remote-account.html .
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-09-28)
+h2(#main). development main (as of 2020-12-10)
+
+"Upgrading from 2.1.0":#v2_1_0
+
+h3. Changes on the collection's @preserve_version@ attribute semantics
+
+The @preserve_version@ attribute on collections was originally designed to allow clients to persist a preexisting collection version. This forced clients to make 2 requests if the intention is to "make this set of changes in a new version that will be kept", so we have changed the semantics to do just that: When passing @preserve_version=true@ along with other collection updates, the current version is persisted and also the newly created one will be persisted on the next update.
+
+h3. System token requirements
+
+System services now log a warning at startup if any of the system tokens (@ManagementToken@, @SystemRootToken@, and @Collections.BlobSigningKey@) are less than 32 characters, or contain characters other than a-z, A-Z, and 0-9. After upgrading, run @arvados-server config-check@ and update your configuration file if needed to resolve any warnings.
+
+The @API.RailsSessionSecretToken@ configuration key has been removed. Delete this entry from your configuration file after upgrading.
+
+h3. Centos7 Python 3 dependency upgraded to python3
+
+Now that Python 3 is part of the base repository in CentOS 7, the Python 3 dependency for Centos7 Arvados packages was changed from SCL rh-python36 to python3.
+
+h2(#v2_1_0). v2.1.0 (2020-10-13)
"Upgrading from 2.0.0":#v2_0_0
--- /dev/null
+---
+layout: default
+navsection: admin
+title: "User activity report"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The @arv-user-activity@ tool generates a summary report of user activity on an Arvados instance based on the audit logs (the @logs@ table).
+
+h2. Installation
+
+h2. Option 1: Install from a distribution package
+
+This installation method is recommended to make the CLI tools available system-wide. It can coexist with the installation method described in option 2, below.
+
+First, configure the "Arvados package repositories":../../install/packages.html
+
+{% assign arvados_component = 'python3-arvados-user-activity' %}
+
+{% include 'install_packages' %}
+
+h2. Option 2: Install from source
+
+Step 1: Check out the arvados source code
+
+Step 2: Change directory to @arvados/tools/user-activity@
+
+Step 3: Run @pip install .@ in an appropriate installation environment, such as a @virtualenv@.
+
+Note: depends on the "Arvados Python SDK":../sdk/python/sdk-python.html and its associated build prerequisites (e.g. @pycurl@).
+
+h2. Usage
+
+Set ARVADOS_API_HOST to the api server of the cluster for which the report should be generated. ARVADOS_API_TOKEN needs to be a "v2 token":../admin/scoped-tokens.html for an admin user, or a superuser token (e.g. generated with @script/create_superuser_token.rb@). Please note that in a login cluster federation, the token needs to be issued by the login cluster, but the report should be generated against the API server of the cluster for which it is desired. In other words, ARVADOS_API_HOST would point at the satellite cluster for which the report is desired, but ARVADOS_API_TOKEN would be a token that belongs to a login cluster user.
+
+Run the tool with the option @--days@ giving the number of days to report on. It will request activity logs from the API and generate a summary report on standard output.
+
+Example run:
+
+<pre>
+$ bin/arv-user-activity --days 14
+User activity on pirca between 2020-11-10 16:42 and 2020-11-24 16:42
+
+Peter Amstutz <peter.amstutz@curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-a4qnxq3pcfcgtkz)
+ organization: "Curii"
+ role: "Software Developer"
+
+ 2020-11-10 16:51-05:00 to 2020-11-11 13:51-05:00 (21:00) Account activity
+ 2020-11-13 13:47-05:00 to 2020-11-14 03:32-05:00 (13:45) Account activity
+ 2020-11-14 04:33-05:00 to 2020-11-15 20:33-05:00 (40:00) Account activity
+ 2020-11-15 21:34-05:00 to 2020-11-16 13:34-05:00 (16:00) Account activity
+ 2020-11-16 16:21-05:00 to 2020-11-16 16:28-05:00 (00:07) Account activity
+ 2020-11-17 15:49-05:00 to 2020-11-17 15:49-05:00 (00:00) Account activity
+ 2020-11-17 15:51-05:00 Created project "New project" (pirca-j7d0g-7bxvkyr4khfa1a4)
+ 2020-11-17 15:51-05:00 Updated project "Test run" (pirca-j7d0g-7bxvkyr4khfa1a4)
+ 2020-11-17 15:51-05:00 Ran container "bwa-mem.cwl container" (pirca-xvhdp-xf2w8dkk17jkk5r)
+ 2020-11-17 15:51-05:00 to 2020-11-17 15:51-05:00 (0:00) Account activity
+ 2020-11-17 15:53-05:00 Ran container "WGS processing workflow scattered over samples container" (pirca-xvhdp-u7bm0wdy6lq4r8k)
+ 2020-11-17 15:53-05:00 to 2020-11-17 15:54-05:00 (00:01) Account activity
+ 2020-11-17 15:55-05:00 Created collection "output for pirca-dz642-36ffk81c8zzopxz" (pirca-4zz18-np35gw690ndzzk7)
+ 2020-11-17 15:55-05:00 to 2020-11-17 15:55-05:00 (0:00) Account activity
+ 2020-11-17 15:55-05:00 Created collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+ 2020-11-17 15:55-05:00 Tagged pirca-4zz18-oiiymetwhnnhhwc
+ 2020-11-17 15:55-05:00 Updated collection "Output of main" (pirca-4zz18-oiiymetwhnnhhwc)
+ 2020-11-17 15:55-05:00 to 2020-11-17 16:04-05:00 (00:09) Account activity
+ 2020-11-17 16:04-05:00 Created collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+ 2020-11-17 16:04-05:00 Tagged pirca-4zz18-f6n9n89e3dhtwvl
+ 2020-11-17 16:04-05:00 Updated collection "Output of main" (pirca-4zz18-f6n9n89e3dhtwvl)
+ 2020-11-17 16:04-05:00 to 2020-11-17 17:55-05:00 (01:51) Account activity
+ 2020-11-17 20:09-05:00 to 2020-11-17 20:09-05:00 (00:00) Account activity
+ 2020-11-17 21:35-05:00 to 2020-11-17 21:35-05:00 (00:00) Account activity
+ 2020-11-18 10:09-05:00 to 2020-11-18 11:00-05:00 (00:51) Account activity
+ 2020-11-18 14:37-05:00 Untagged pirca-4zz18-st8yzjan1nhxo1a
+ 2020-11-18 14:37-05:00 Deleted collection "Output of main" (pirca-4zz18-st8yzjan1nhxo1a)
+ 2020-11-18 17:44-05:00 to 2020-11-18 17:44-05:00 (00:00) Account activity
+ 2020-11-19 12:18-05:00 to 2020-11-19 12:19-05:00 (00:01) Account activity
+ 2020-11-19 13:57-05:00 to 2020-11-19 14:21-05:00 (00:24) Account activity
+ 2020-11-20 09:48-05:00 to 2020-11-20 22:51-05:00 (13:03) Account activity
+ 2020-11-20 23:52-05:00 to 2020-11-22 22:32-05:00 (46:40) Account activity
+ 2020-11-22 23:37-05:00 to 2020-11-23 13:52-05:00 (14:15) Account activity
+ 2020-11-23 14:53-05:00 to 2020-11-24 11:58-05:00 (21:05) Account activity
+ 2020-11-24 15:06-05:00 to 2020-11-24 16:38-05:00 (01:32) Account activity
+
+Marc Rubenfield <mrubenfield@curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-v9s9q97pgydh1yf)
+ 2020-11-11 12:27-05:00 Untagged pirca-4zz18-xmq257bsla4kdco
+ 2020-11-11 12:27-05:00 Deleted collection "Output of main" (pirca-4zz18-xmq257bsla4kdco)
+
+Ward Vandewege <ward@curii.com> (https://workbench.pirca.arvadosapi.com/users/jutro-tpzed-9z6foyez9ydn2hl)
+ organization: "Curii Corporation, Inc."
+ organization_email: "ward@curii.com"
+ role: "System Administrator"
+ website_url: "https://curii.com"
+
+ 2020-11-19 19:30-05:00 to 2020-11-19 19:46-05:00 (00:16) Account activity
+ 2020-11-20 10:51-05:00 to 2020-11-20 11:26-05:00 (00:35) Account activity
+ 2020-11-24 12:01-05:00 to 2020-11-24 13:01-05:00 (01:00) Account activity
+</pre>
--- /dev/null
+---
+layout: default
+navsection: api
+navmenu: API Methods
+title: "S3 API"
+
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The Simple Storage Service (S3) API is a de-facto standard for object storage originally developed by Amazon Web Services. Arvados supports accessing files in Keep using the S3 API.
+
+S3 is supported by many "cloud native" applications, and client libraries exist in many languages for programmatic access.
+
+h3. Endpoints and Buckets
+
+To access Arvados S3 using an S3 client library, you must tell it to use the URL of the keep-web server (this is @Services.WebDAVDownload.ExternalURL@ in the public configuration) as the custom endpoint. The keep-web server will decide to treat it as an S3 API request based on the presence of an AWS-format Authorization header. Requests without an Authorization header, or differently formatted Authorization, will be treated as "WebDAV":keep-webdav.html .
+
+The "bucket name" is an Arvados collection uuid, portable data hash, or project uuid.
+
+Path-style and virtual host-style requests are supported.
+* A path-style request uses the hostname indicated by @Services.WebDAVDownload.ExternalURL@, with the bucket name in the first path segment: @https://download.example.com/zzzzz-4zz18-asdfgasdfgasdfg/@.
+* A virtual host-style request uses the hostname pattern indicated by @Services.WebDAV.ExternalURL@, with a bucket name in place of the leading @*@: @https://zzzzz-4zz18-asdfgasdfgasdfg.collections.example.com/@.
+
+If you have wildcard DNS, TLS, and routing set up, an S3 client configured with endpoint @collections.example.com@ should work regardless of which request style it uses.
+
+h3. Supported Operations
+
+h4. ListObjects
+
+Supports the following request query parameters:
+
+* delimiter
+* marker
+* max-keys
+* prefix
+
+h4. GetObject
+
+Supports the @Range@ header.
+
+h4. PutObject
+
+Can be used to create or replace a file in a collection.
+
+An empty PUT with a trailing slash and @Content-Type: application/x-directory@ will create a directory within a collection if Arvados configuration option @Collections.S3FolderObjects@ is true.
+
+Missing parent/intermediate directories within a collection are created automatically.
+
+Cannot be used to create a collection or project.
+
+h4. DeleteObject
+
+Can be used to remove files from a collection.
+
+If used on a directory marker, it will delete the directory only if the directory is empty.
+
+h4. HeadBucket
+
+Can be used to determine if a bucket exists and if client has read access to it.
+
+h4. HeadObject
+
+Can be used to determine if an object exists and if client has read access to it.
+
+h4. GetBucketVersioning
+
+Bucket versioning is presently not supported, so this will always respond that bucket versioning is not enabled.
+
+h3. Authorization mechanisms
+
+Keep-web accepts AWS Signature Version 4 (AWS4-HMAC-SHA256) as well as the older V2 AWS signature.
+
+If your client uses V4 signatures exclusively _and_ your Arvados token was issued by the same cluster you are connecting to, you can use the Arvados token's UUID part as your S3 Access Key, and its secret part as your S3 Secret Key. This is preferred, where applicable.
+
+Example using cluster @zzzzz@:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @zzzzz-gj3su-yyyyyyyyyyyyyyy@
+* Secret Key: @xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+
+In all other cases, replace every @/@ character in your Arvados token with @_@, and use the resulting string as both Access Key and Secret Key.
+
+Example using a cluster other than @zzzzz@ _or_ an S3 client that uses V2 signatures:
+* Arvados token: @v2/zzzzz-gj3su-yyyyyyyyyyyyyyy/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Access Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
+* Secret Key: @v2_zzzzz-gj3su-yyyyyyyyyyyyyyy_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@
--- /dev/null
+---
+layout: default
+navsection: api
+navmenu: API Methods
+title: "Keep-web URL patterns"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Files served by @keep-web@ can be rendered directly in the browser, or @keep-web@ can instruct the browser to only download the file.
+
+When serving files that will render directly in the browser, it is important to properly configure the keep-web service to migitate cross-site-scripting (XSS) attacks. A HTML page can be stored in a collection. If an attacker causes a victim to visit that page through Workbench, the HTML will be rendered by the browser. If all collections are served at the same domain, the browser will consider collections as coming from the same origin, which will grant access to the same browsing data (cookies and local storage). This would enable malicious Javascript on that page to access Arvados on behalf of the victim.
+
+This can be mitigated by having separate domains for each collection, or limiting preview to circumstances where the collection is not accessed with the user's regular full-access token. For cluster administrators that understand the risks, this protection can also be turned off.
+
+The following "same origin" URL patterns are supported for public collections and collections shared anonymously via secret links (i.e., collections which can be served by keep-web without making use of any implicit credentials like cookies). See "Same-origin URLs" below.
+
+<pre>
+http://collections.example.com/c=uuid_or_pdh/path/file.txt
+http://collections.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
+</pre>
+
+The following "multiple origin" URL patterns are supported for all collections:
+
+<pre>
+http://uuid_or_pdh--collections.example.com/path/file.txt
+http://uuid_or_pdh--collections.example.com/t=TOKEN/path/file.txt
+</pre>
+
+In the "multiple origin" form, the string @--@ can be replaced with @.@ with identical results (assuming the downstream proxy is configured accordingly). These two are equivalent:
+
+<pre>
+http://uuid_or_pdh--collections.example.com/path/file.txt
+http://uuid_or_pdh.collections.example.com/path/file.txt
+</pre>
+
+The first form (with @--@ instead of @.@) avoids the cost and effort of deploying a wildcard TLS certificate for @*.collections.example.com@ at sites that already have a wildcard certificate for @*.example.com@ . The second form is likely to be easier to configure, and more efficient to run, on a downstream proxy.
+
+In all of the above forms, the @collections.example.com@ part can be anything at all: keep-web itself ignores everything after the first @.@ or @--@. (Of course, in order for clients to connect at all, DNS and any relevant proxies must be configured accordingly.)
+
+In all of the above forms, the @uuid_or_pdh@ part can be either a collection UUID or a portable data hash with the @+@ character optionally replaced by @-@ . (When @uuid_or_pdh@ appears in the domain name, replacing @+@ with @-@ is mandatory, because @+@ is not a valid character in a domain name.)
+
+In all of the above forms, a top level directory called @_@ is skipped. In cases where the @path/file.txt@ part might start with @t=@ or @c=@ or @_/@, links should be constructed with a leading @_/@ to ensure the top level directory is not interpreted as a token or collection ID.
+
+Assuming there is a collection with UUID @zzzzz-4zz18-znfnqtbbv4spc3w@ and portable data hash @1f4b0bc7583c2a7f9102c395f4ffc5e3+45@, the following URLs are interchangeable:
+
+<pre>
+http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
+http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
+http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
+</pre>
+
+The following URLs are read-only, but will return the same content as above:
+
+<pre>
+http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
+http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
+http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt
+http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt
+</pre>
+
+If the collection is named "MyCollection" and located in a project called "MyProject" which is in the home project of a user with username is "bob", the following read-only URL is also available when authenticating as bob:
+
+pre. http://collections.example.com/users/bob/MyProject/MyCollection/foo/bar.txt
+
+An additional form is supported specifically to make it more convenient to maintain support for existing Workbench download links:
+
+pre. http://collections.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt
+
+A regular Workbench "download" link is also accepted, but credentials passed via cookie, header, etc. are ignored. Only public data can be served this way:
+
+pre. http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
--- /dev/null
+---
+layout: default
+navsection: api
+navmenu: API Methods
+title: "WebDAV"
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+"Web Distributed Authoring and Versioning (WebDAV)":https://tools.ietf.org/html/rfc4918 is an IETF standard set of extensions to HTTP to manipulate and retrieve hierarchical web resources, similar to directories in a file system. Arvados supports accessing files in Keep using WebDAV.
+
+Most major operating systems include built-in support for mounting WebDAV resources as network file systems, see user guide sections for "Windows":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-windows.html , "macOS":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-os-x.html , "Linux (Gnome)":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html#gnome . WebDAV is also supported by various standalone storage browser applications such as "Cyberduck":https://cyberduck.io/ and client libraries exist in many languages for programmatic access.
+
+Keep-web provides read/write HTTP (WebDAV) access to files stored in Keep. It serves public data to anonymous and unauthenticated clients, and serves private data to clients that supply Arvados API tokens.
+
+h3. Supported Operations
+
+Supports WebDAV HTTP methods @GET@, @PUT@, @DELETE@, @PROPFIND@, @COPY@, and @MOVE@.
+
+Does not support @LOCK@ or @UNLOCK@. These methods will be accepted, but are no-ops.
+
+h3. Browsing
+
+Requests can be authenticated a variety of ways as described below in "Authentication mechanisms":#auth . An unauthenticated request will return a 401 Unauthorized response with a @WWW-Authenticate@ header indicating "support for RFC 7617 Basic Authentication":https://tools.ietf.org/html/rfc7617 .
+
+Getting a listing from keep-web starting at the root path @/@ will return two folders, @by_id@ and @users@.
+
+The @by_id@ folder will return an empty listing. However, a path which starts with /by_id/ followed by a collection uuid, portable data hash, or project uuid will return the listing of that object.
+
+The @users@ folder will return a listing of the users for whom the client has permission to read the "home" project of that user. Browsing an individual user will return the collections and projects directly owned by that user. Browsing those collections and projects return listings of the files, directories, collections, and subprojects they contain, and so forth.
+
+In addition to the @/by_id/@ path prefix, the collection or project can be specified using a path prefix of @/c=<uuid or pdh>/@ or (if the cluster is properly configured) as a virtual host. This is described on "Keep-web URLs":keep-web-urls.html
+
+h3(#auth). Authentication mechanisms
+
+A token can be provided in an Authorization header as a @Bearer@ token:
+
+<pre>
+Authorization: Bearer o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+</pre>
+
+A token can also be provided with "RFC 7617 Basic Authentication":https://tools.ietf.org/html/rfc7617 in this case, the payload is formatted as @username:token@ and encoded with base64. The username must be non-empty, but is ignored. In this example, the username is "user":
+
+<pre>
+Authorization: Basic dXNlcjpvMDdqNHB4N1JsSks0Q3VNWXA3QzBMRFQ0Q3pSMUoxcUJFNUF2bzdlQ2NVak9UaWt4Swo=
+</pre>
+
+A base64-encoded token can be provided in a cookie named "api_token":
+
+<pre>
+Cookie: api_token=bzA3ajRweDdSbEpLNEN1TVlwN0MwTERUNEN6UjFKMXFCRTVBdm83ZUNjVWpPVGlreEs=
+</pre>
+
+A token can be provided in an URL-encoded query string:
+
+<pre>
+GET /foo/bar.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+</pre>
+
+A token can be provided in a URL-encoded path (as described in the previous section):
+
+<pre>
+GET /t=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK/_/foo/bar.txt
+</pre>
+
+A suitably encoded token can be provided in a POST body if the request has a content type of application/x-www-form-urlencoded or multipart/form-data:
+
+<pre>
+POST /foo/bar.txt
+Content-Type: application/x-www-form-urlencoded
+[...]
+api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
+</pre>
+
+If a token is provided in a query string or in a POST request, the response is an HTTP 303 redirect to an equivalent GET request, with the token stripped from the query string and added to a cookie instead.
+
+h3. Indexes
+
+Keep-web returns a generic HTML index listing when a directory is requested with the GET method. It does not serve a default file like "index.html". Directory listings are also returned for WebDAV PROPFIND requests.
+
+h3. Range requests
+
+Keep-web supports partial resource reads using the HTTP @Range@ header as specified in "RFC 7233":https://tools.ietf.org/html/rfc7233 .
+
+h3. Compatibility
+
+Client-provided authorization tokens are ignored if the client does not provide a @Host@ header.
+
+In order to use the query string or a POST form authorization mechanisms, the client must follow 303 redirects; the client must accept cookies with a 303 response and send those cookies when performing the redirect; and either the client or an intervening proxy must resolve a relative URL ("//host/path") if given in a response Location header.
+
+h3. Intranet mode
+
+Normally, Keep-web accepts requests for multiple collections using the same host name, provided the client's credentials are not being used. This provides insufficient XSS protection in an installation where the "anonymously accessible" data is not truly public, but merely protected by network topology.
+
+In such cases -- for example, a site which is not reachable from the internet, where some data is world-readable from Arvados's perspective but is intended to be available only to users within the local network -- the downstream proxy should configured to return 401 for all paths beginning with "/c=".
+
+h3. Same-origin URLs
+
+Without the same-origin protection outlined above, a web page stored in collection X could execute JavaScript code that uses the current viewer's credentials to download additional data from collection Y -- data which is accessible to the current viewer, but not to the author of collection X -- from the same origin (``https://collections.example.com/'') and upload it to some other site chosen by the author of collection X.
|limit |integer|Maximum number of resources to return. If not provided, server will provide a default limit. Server may also impose a maximum number of records that can be returned in a single request.|query|
|offset |integer|Skip the first 'offset' number of resources that would be returned under the given filter conditions.|query|
|filters |array |"Conditions for selecting resources to return.":#filters|query|
-|order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order.
+|order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order. (If not specified, it will be ascending).
Example: @["head_uuid asc","modified_at desc"]@
-Default: @["created_at desc"]@|query|
+Default: @["modified_at desc", "uuid asc"]@|query|
|select |array |Set of attributes to include in the response.
Example: @["head_uuid","tail_uuid"]@
Default: all available attributes. As a special case, collections do not return "manifest_text" unless explicitly selected.|query|
|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
-|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["main","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
|is_trashed|boolean|True if @trash_at@ is in the past, false if not.||
|current_version_uuid|string|UUID of the collection's current version. On new collections, it'll be equal to the @uuid@ attribute.||
|version|number|Version number, starting at 1 on new collections. This attribute is read-only.||
-|preserve_version|boolean|When set to true on a current version, it will be saved on the next versionable update.||
+|preserve_version|boolean|When set to true on a current version, it will be persisted. When passing @true@ as part of a bigger update call, both current and newly created versions are persisted.||
|file_count|number|The total number of files in the collection. This attribute is read-only.||
|file_size_total|number|The sum of the file sizes in the collection. This attribute is read-only.||
|recursive|boolean (default false)|Include items owned by subprojects.|query|@true@|
|exclude_home_project|boolean (default false)|Only return items which are visible to the user but not accessible within the user's home project. Use this to get a list of items that are shared with the user. Uses the logic described under the "shared" endpoint.|query|@true@|
|include|string|If provided with the value "owner_uuid", this will return owner objects in the "included" field of the response.|query||
+|include_trash|boolean (default false)|Include trashed objects.|query|@true@|
+|include_old_versions|boolean (default false)|Include past versions of the collections being listed.|query|@true@|
Notes:
h3(#script_version). Specifying Git versions
-The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
+The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, <code>HEAD@{1}</code> refers to the local reflog, and @origin/main@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version.
h3. Runtime constraints
h4. Examples
-Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "master" commit. This works irrespective of whether the previous job was submitted using the name "master", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
+Run the script "crunch_scripts/hash.py" in the repository "you" using the "main" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "main" commit. This works irrespective of whether the previous job was submitted using the name "main", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "master" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "master" commit.
+Arvados should re-use a previous job if the "script_version" of the previous job is between "earlier_version_tag" and the "main" commit (inclusive), but not the commit indicated by "blacklisted_version_tag". If there are no previous jobs matching these criteria, run the job using the "main" commit.
<notextile><pre>
{
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
"job": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
}
}
</pre></notextile>
-Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "master" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
+Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using the current "main" commit. Because it is marked as "nondeterministic", this job will not be considered as a suitable candidate for future job submissions that use the "find_or_create" feature.
<notextile><pre>
{
"job": {
"script": "monte-carlo.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"nondeterministic": true,
"script_parameters": {
"input": "c1bad4b39ca5a924e481008009d94e32+210"
"do_hash": {
"script": "hash.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"required": true,
"filter": {
"script": "0-filter.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "do_hash"
"cat_in_the_hat": {
"script": "cat.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"input": {
"output_of": "cat_in_the_hat"
"thing1": {
"script": "thing1.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"thing2": {
"script": "thing2.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": { }
},
"cleanup": {
"script": "cleanup.py",
"repository": "<b>you</b>/<b>you</b>",
- "script_version": "master",
+ "script_version": "main",
"script_parameters": {
"mess1": {
"output_of": "thing1"
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
h2. Shut down
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
<pre>
$ helm del arvados
</pre>
This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2. Prerequisites
h3. Install tooling
h2. Shut down
{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+This Helm chart uses Kubernetes <i>persistent volumes</i> for the Postgresql and Keepstore data volumes. These volumes will be retained after you delete the Arvados helm chart with the command below. Because those volumes are stored in the local Minikube Kubernetes cluster, if you delete that cluster (e.g. with <i>minikube delete</i>) the Kubernetes persistent volumes will also be deleted.
{% include 'notebox_end' %}
<pre>
Arvados on Kubernetes is implemented as a @Helm 3@ chart.
-{% include 'notebox_begin_warning' %}
-This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
h2(#overview). Overview
This Helm chart provides a basic, small Arvados cluster.
The certificate will be added under the "Arvados testing" organization as "arvbox testing root CA".
-To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage (instructions for Debian/Ubuntu):
+To access your Arvbox instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
-# copy @arvbox-root-cert.pem@ to @/usr/local/share/ca-certificates/@
-# run @/usr/sbin/update-ca-certificates@
+h3. On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvbox-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+h3. On CentOS:
+
+<notextile>
+<pre><code>cp arvbox-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
h2. Configs
h2(#introduction). Introduction
-This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html . For information on installing Slurm, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html
+This assumes you already have a Slurm cluster, and have "set up all of your compute nodes":install-compute-node.html. Slurm packages are available for CentOS, Debian and Ubuntu. Please see your distribution package repositories. For information on installing Slurm from source, see "this install guide":https://slurm.schedmd.com/quickstart_admin.html
The Arvados Slurm dispatcher can run on any node that can submit requests to both the Arvados API server and the Slurm controller (via @sbatch@). It is not resource-intensive, so you can run it on the API server node.
<div class="offset1">
table(table table-bordered table-condensed).
|||\5=. Appropriate for|
-||_. Ease of setup|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
+||_. Setup difficulty|_. Multiuser/networked access|_. Workflow Development and Testing|_. Large Scale Production|_. Development of Arvados|_. Arvados Evaluation|
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
+|"Installation with Salt":salt-single-host.html (single host)|Easy|no|yes|no|yes|yes|
+|"Installation with Salt":salt-multi-host.html (multi host)|Moderate|yes|yes|yes|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
-|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
+|"Manual installation":install-manual-prerequisites.html|Hard|yes|yes|yes|no|no|
|"Cluster Operation Subscription supported by Curii":mailto:info@curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
<notextile>
<pre><code> SystemRootToken: <span class="userinput">"$system_root_token"</span>
ManagementToken: <span class="userinput">"$management_token"</span>
- API:
- RailsSessionSecretToken: <span class="userinput">"$rails_secret_token"</span>
Collections:
- BlobSigningKey: <span class="userinput">"blob_signing_key"</span>
+ BlobSigningKey: <span class="userinput">"$blob_signing_key"</span>
</code></pre>
</notextile>
-@SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+These secret tokens are used to authenticate messages between Arvados components.
+* @SystemRootToken@ is used by Arvados system services to authenticate as the system (root) user when communicating with the API server.
+* @ManagementToken@ is used to authenticate access to system metrics.
+* @Collections.BlobSigningKey@ is used to control access to Keep blocks.
-@ManagementToken@ is used to authenticate access to system metrics.
-
-@API.RailsSessionSecretToken@ is required by the API server.
-
-@Collections.BlobSigningKey@ is used to control access to Keep blocks.
-
-You can generate a random token for each of these items at the command line like this:
+Each token should be a string of at least 50 alphanumeric characters. You can generate a suitable token with the following command:
<notextile>
-<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50; echo</span>
+<pre><code>~$ <span class="userinput">tr -dc 0-9a-zA-Z </dev/urandom | head -c50 ; echo</span>
</code></pre>
</notextile>
+++ /dev/null
----
-layout: default
-navsection: installguide
-title: Sample compute node ping script
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-When a new elastic compute node is booted, it needs to contact Arvados to register itself. Here is an example ping script to run on boot.
-
-<notextile> {% code 'compute_ping_rb' as ruby %} </notextile>
h2(#introduction). Introduction
-The Keep-web server provides read/write HTTP (WebDAV) access to files stored in Keep. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
+The Keep-web server provides read/write access to files stored in Keep using WebDAV and S3 protocols. This makes it easy to access files in Keep from a browser, or mount Keep as a network folder using WebDAV support in various operating systems. It serves public data to unauthenticated clients, and serves private data to clients that supply Arvados API tokens. It can be installed anywhere with access to Keep services, typically behind a web proxy that provides TLS support. See the "godoc page":http://godoc.org/github.com/curoverse/arvados/services/keep-web for more detail.
h2(#dns). Configure DNS
</code></pre>
</notextile>
+This option is preferred if you plan to access Keep using third-party S3 client software, because it accommodates S3 virtual host-style requests and path-style requests without any special client configuration.
+
h4. Under the main domain
Alternately, they can go under the main domain by including @--@:
|_. Distribution|_. State|_. Last supported version|
|CentOS 7|Supported|Latest|
|Debian 10 ("buster")|Supported|Latest|
-|Debian 9 ("stretch")|Supported|Latest|
|Ubuntu 18.04 ("bionic")|Supported|Latest|
|Ubuntu 16.04 ("xenial")|Supported|Latest|
-|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
+|Debian 9 ("stretch")|EOL|Latest 2.1.X release|
|Debian 8 ("jessie")|EOL|1.4.3|
+|Ubuntu 14.04 ("trusty")|EOL|1.4.3|
|Ubuntu 12.04 ("precise")|EOL|8ed7b6dd5d4df93a3f37096afe6d6f81c2a7ef6e (2017-05-03)|
|Debian 7 ("wheezy")|EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
|CentOS 6 |EOL|997479d1408139e96ecdb42a60b4f727f814f6c9 (2016-12-28)|
{% include 'note_python_sc' %}
# Install PostgreSQL
- <notextile><pre># <span class="userinput">yum install rh-postgresql95 rh-postgresql95-postgresql-contrib</span>
-~$ <span class="userinput">scl enable rh-postgresql95 bash</span></pre></notextile>
+ <notextile><pre># <span class="userinput">yum install rh-postgresql12 rh-postgresql12-postgresql-contrib</span>
+~$ <span class="userinput">scl enable rh-postgresql12 bash</span></pre></notextile>
# Initialize the database
<notextile><pre># <span class="userinput">postgresql-setup initdb</span></pre></notextile>
# Configure the database to accept password connections
<notextile><pre><code># <span class="userinput">sed -ri -e 's/^(host +all +all +(127\.0\.0\.1\/32|::1\/128) +)ident$/\1md5/' /var/lib/pgsql/data/pg_hba.conf</span></code></pre></notextile>
# Configure the database to launch at boot and start now
- <notextile><pre># <span class="userinput">systemctl enable --now rh-postgresql95-postgresql</span></pre></notextile>
+ <notextile><pre># <span class="userinput">systemctl enable --now rh-postgresql12-postgresql</span></pre></notextile>
h3(#debian). Debian or Ubuntu
-Debian 8 (Jessie) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
-
-Ubuntu 14.04 (Trusty) requires an updated PostgreSQL version, see "the PostgreSQL ubuntu repository":https://www.postgresql.org/download/linux/ubuntu/
+Debian 10 (Buster) and Ubuntu 16.04 (Xenial) and later versions include a sufficiently recent version of Postgres.
# Install PostgreSQL
<notextile><pre># <span class="userinput">apt-get --no-install-recommends install postgresql postgresql-contrib</span></pre></notextile>
table(table table-bordered table-condensed).
|_. OS version|_. Command|
-|Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ buster main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Debian 9 ("stretch")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ stretch main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ bionic main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
-|Ubuntu 16.04 ("xenial")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/ xenial main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Debian 10 ("buster")|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/buster buster main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 20.04 ("focal")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/focal focal main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
+|Ubuntu 18.04 ("bionic")[1]|<notextile><code><span class="userinput">echo "deb http://apt.arvados.org/bionic bionic main" | tee /etc/apt/sources.list.d/arvados.list</span></code></notextile>|
{% include 'notebox_begin' %}
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Multi host Arvados
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Install Saltstack":#saltstack
+# "Install dependencies":#dependencies
+# "Install Arvados using Saltstack":#saltstack
+# "DNS configuration":#final_steps
+# "Initial user and login":#initial_user
+
+h2(#saltstack). Install Saltstack
+
+If you already have a Saltstack environment you can skip this section.
+
+The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
+
+<notextile>
+<pre><code>curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+</code></pre>
+</notextile>
+
+For more information check "Saltstack's documentation":https://docs.saltstack.com/en/latest/topics/installation/index.html
+
+h2(#dependencies). Install dependencies
+
+Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be installed using their respective Saltstack formulas.
+
+The formulas we use are:
+
+* "postgres":https://github.com/saltstack-formulas/postgres-formula.git
+* "nginx":https://github.com/saltstack-formulas/nginx-formula.git
+* "docker":https://github.com/saltstack-formulas/docker-formula.git
+* "locale":https://github.com/saltstack-formulas/locale-formula.git
+
+There are example Salt pillar files for each of those formulas in the "arvados-formula's test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples directory. As they are, they allow you to get all the main Arvados components up and running.
+
+h2(#saltstack). Install Arvados using Saltstack
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+The Arvados formula we maintain is located in the Saltstack's community repository of formulas:
+
+* "arvados-formula":https://github.com/saltstack-formulas/arvados-formula.git
+
+The @development@ version lives in our own repository
+
+* "arvados-formula development":https://github.com/arvados/arvados-formula.git
+
+This last one might break from time to time, as we try and add new features. Use with caution.
+
+As much as possible, we try to keep it up to date, with example pillars to help you deploy Arvados.
+
+For those familiar with Saltstack, the process to get it deployed is similar to any other formula:
+
+1. Fork/copy the formula to your Salt master host.
+2. Edit the Arvados, nginx, postgres, locale and docker pillars to match your desired configuration.
+3. Run a @state.apply@ to get it deployed.
+
+h2(#final_steps). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster's nodes.
+
+The simplest way to do this is to add entries in the @/etc/hosts@ file of every host:
+
+<notextile>
+<pre><code>export CLUSTER="arva2"
+export DOMAIN="arv.local"
+
+echo A.B.C.a api ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.b keep keep.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.c keep0 keep0.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.d collections collections.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.e download download.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.f ws ws.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.g workbench workbench.${CLUSTER}.${DOMAIN} >> /etc/hosts
+echo A.B.C.h workbench2 workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+</code></pre>
+</notextile>
+
+Replacing in each case de @A.B.C.x@ IP with the corresponding IP of the node.
+
+If your infrastructure uses another DNS service setup, add the corresponding entries accordingly.
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you did not change the defaults, the initial URL will be:
+
+* https://workbench.arva2.arv.local
+
+or, in general, the url format will be:
+
+* https://workbench.@<cluster>.<domain>@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change the defaults, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Single host Arvados
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Install Saltstack":#saltstack
+# "Single host install using the provision.sh script":#single_host
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
+# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
+
+h2(#saltstack). Install Saltstack
+
+If you already have a Saltstack environment you can skip this section.
+
+The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
+
+<notextile>
+<pre><code>curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+</code></pre>
+</notextile>
+
+For more information check "Saltstack's documentation":https://docs.saltstack.com/en/latest/topics/installation/index.html
+
+h2(#single_host). Single host install using the provision.sh script
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+Use the @provision.sh@ script to deploy Arvados, which is implemented with the @arvados-formula@ in a Saltstack master-less setup:
+
+* edit the variables at the very beginning of the file,
+* run the script as root
+* wait for it to finish
+
+This will install all the main Arvados components to get you up and running. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host and your network bandwidth. On a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
+
+If everything goes OK, you'll get some final lines stating something like:
+
+<notextile>
+<pre><code>arvados: Succeeded: 109 (changed=9)
+arvados: Failed: 0
+</code></pre>
+</notextile>
+
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster.
+
+The simplest way to do this is to edit your @/etc/hosts@ file (as root):
+
+<notextile>
+<pre><code>export CLUSTER="arva2"
+export DOMAIN="arv.local"
+export HOST_IP="127.0.0.2" # This is valid either if installing in your computer directly
+ # or in a Vagrant VM. If you're installing it on a remote host
+ # just change the IP to match that of the host.
+echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+</code></pre>
+</notextile>
+
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so you can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button. Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+* On CentOS:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you changed nothing in the @provision.sh@ script, the initial URL will be:
+
+* https://workbench.arva2.arv.local
+
+or, in general, the url format will be:
+
+* https://workbench.@<cluster>.<domain>@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change these values in the @provision.sh@ script, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+The @provision.sh@ script saves a simple example test workflow in the @/tmp/cluster_tests@. If you want to run it, just change to that directory and run:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
+
+It will create a test user, upload a small workflow and run it. If everything goes OK, the output should similar to this (some output was shortened for clarity):
+
+<notextile>
+<pre><code>Creating Arvados Standard Docker Images project
+Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
+{
+ ...
+ "uuid":"arva2-o0j2j-n4zu4cak5iifq2a",
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+}
+Uploading arvados/jobs' docker image to the project
+2.1.1: Pulling from arvados/jobs
+8559a31e96f4: Pulling fs layer
+...
+Status: Downloaded newer image for arvados/jobs:2.1.1
+docker.io/arvados/jobs:2.1.1
+2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
+2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
+arva2-4zz18-1u5pvbld7cvxuy2
+Creating initial user ('admin')
+Setting up user ('admin')
+{
+ "items":[
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1ownrdne0ok9iox"
+ },
+ {
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "uuid":"arva2-o0j2j-1zbeyhcwxc1tvb7"
+ },
+ {
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "owner_uuid":"arva2-tpzed-000000000000000",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+ }
+ ],
+ "kind":"arvados#HashList"
+}
+Activating user 'admin'
+{
+ ...
+ "email":"admin@arva2.arv.local",
+ ...
+ "username":"admin",
+ "uuid":"arva2-tpzed-3wrm93zmzpshrq2",
+ ...
+}
+Running test CWL workflow
+INFO /usr/bin/cwl-runner 2.1.1, arvados-python-client 2.1.1, cwltool 3.0.20200807132242
+INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl'
+...
+INFO Using cluster arva2 (https://arva2.arv.local:8443/)
+INFO Upload local files: "test.txt"
+INFO Uploaded to ea34d971b71d5536b4f6b7d6c69dc7f6+50 (arva2-4zz18-c8uvwqdry4r8jao)
+INFO Using collection cache size 256 MiB
+INFO [container hasher-workflow.cwl] submitted container_request arva2-xvhdp-v1bkywd58gyocwm
+INFO [container hasher-workflow.cwl] arva2-xvhdp-v1bkywd58gyocwm is Final
+INFO Overall process status is success
+INFO Final output collection d6c69a88147dde9d52a418d50ef788df+123
+{
+ "hasher_out": {
+ "basename": "hasher3.md5sum.txt",
+ "class": "File",
+ "location": "keep:d6c69a88147dde9d52a418d50ef788df+123/hasher3.md5sum.txt",
+ "size": 95
+ }
+}
+INFO Final process status is success
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Arvados in a VM with Vagrant
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Vagrant":#vagrant
+# "Final steps":#final_steps
+## "DNS configuration":#dns_configuration
+## "Install root certificate":#ca_root_certificate
+# "Initial user and login":#initial_user
+# "Test the installed cluster running a simple workflow":#test_install
+
+h2(#vagrant). Vagrant
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+A @Vagrantfile@ is provided to install Arvados in a virtual machine on your computer using "Vagrant":https://www.vagrantup.com/.
+
+To get it running, install Vagrant in your computer, edit the variables at the top of the @provision.sh@ script as needed, and run
+
+<notextile>
+<pre><code>vagrant up
+</code></pre>
+</notextile>
+
+If you want to reconfigure the running box, you can just:
+
+1. edit the pillars to suit your needs
+2. run
+
+<notextile>
+<pre><code>vagrant reload --provision
+</code></pre>
+</notextile>
+
+h2(#final_steps). Final configuration steps
+
+h3(#dns_configuration). DNS configuration
+
+After the setup is done, you need to set up your DNS to be able to access the cluster.
+
+The simplest way to do this is to edit your @/etc/hosts@ file (as root):
+
+<notextile>
+<pre><code>export CLUSTER="arva2"
+export DOMAIN="arv.local"
+export HOST_IP="127.0.0.2" # This is valid either if installing in your computer directly
+ # or in a Vagrant VM. If you're installing it on a remote host
+ # just change the IP to match that of the host.
+echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
+</code></pre>
+</notextile>
+
+h3(#ca_root_certificate). Install root certificate
+
+Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+
+For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so you can add it to your workstation.
+
+Installing the root certificate into your web browser will prevent security errors when accessing Arvados services with your web browser.
+
+# Go to the certificate manager in your browser.
+#* In Chrome, this can be found under "Settings → Advanced → Manage Certificates" or by entering @chrome://settings/certificates@ in the URL bar.
+#* In Firefox, this can be found under "Preferences → Privacy & Security" or entering @about:preferences#privacy@ in the URL bar and then choosing "View Certificates...".
+# Select the "Authorities" tab, then press the "Import" button. Choose @arvados-snakeoil-ca.pem@
+
+The certificate will be added under the "Arvados Formula".
+
+To access your Arvados instance using command line clients (such as arv-get and arv-put) without security errors, install the certificate into the OS certificate storage.
+
+* On Debian/Ubuntu:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /usr/local/share/ca-certificates/
+/usr/sbin/update-ca-certificates
+</code></pre>
+</notextile>
+
+* On CentOS:
+
+<notextile>
+<pre><code>cp arvados-root-cert.pem /etc/pki/ca-trust/source/anchors/
+/usr/bin/update-ca-trust
+</code></pre>
+</notextile>
+
+h2(#initial_user). Initial user and login
+
+At this point you should be able to log into the Arvados cluster.
+
+If you didn't change the defaults, the initial URL will be:
+
+* https://workbench.arva2.arv.local:8443
+
+or, in general, the url format will be:
+
+* https://workbench.@<cluster>.<domain>:8443@
+
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
+
+Assuming you didn't change the defaults, the initial credentials are:
+
+* User: 'admin'
+* Password: 'password'
+* Email: 'admin@arva2.arv.local'
+
+h2(#test_install). Test the installed cluster running a simple workflow
+
+As documented in the <a href="{{ site.baseurl }}/install/salt-single-host.html">Single Host installation</a> page, You can run a test workflow to verify the installation finished correctly. To do so, you can follow these steps:
+
+<notextile>
+<pre><code>vagrant ssh</code></pre>
+</notextile>
+
+and once in the instance:
+
+<notextile>
+<pre><code>cd /tmp/cluster_tests
+./run-test.sh
+</code></pre>
+</notextile>
--- /dev/null
+---
+layout: default
+navsection: installguide
+title: Salt prerequisites
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Introduction":#introduction
+# "Choose an installation method":#installmethod
+
+h2(#introduction). Introduction
+
+To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://github.com/saltstack-formulas/arvados-formula which can help you get an Arvados cluster up and running.
+
+Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a master/minion setup or master-less.
+
+This is a package-based installation method. The Salt scripts are available from the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory in the Arvados git repository.
+
+h2(#installmethod). Choose an installation method
+
+The salt formulas can be used in different ways. Choose one of these three options to install Arvados:
+
+* "Use Vagrant to install Arvados in a virtual machine":salt-vagrant.html
+* "Arvados on a single host":salt-single-host.html
+* "Arvados across multiple hosts":salt-multi-host.html
# "Install Ruby":../../install/ruby.html
# "Install the Python SDK":../python/sdk-python.html
-The SDK uses @curl@ which depends on the @libcurl@ C library. To build the module you may have to install additional packages. On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library. To build the module you may have to install additional packages. On Debian 10 this is:
<pre>
$ apt-get install build-essential libcurl4-openssl-dev
Delete a group
@arv group delete --uuid 6dnxa-j7d0g-iw7i6n43d37jtog@
+Create an empty collection
+@arv collection create --collection '{"name": "test collection"}'@
h3. Common commands
---
layout: default
navsection: sdk
-navmenu: Python
+navmenu: Go
title: Examples
...
{% comment %}
You can save this source as a .go file and run it:
-<notextile>{% code 'example_sdk_go' as go %}</notextile>
+<notextile>{% code example_sdk_go as go %}</notextile>
A few more usage examples can be found in the "services/keepproxy":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/services/keepproxy and "sdk/go/keepclient":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/sdk/go/keepclient directories in the arvados source tree.
Note:
-The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 9 this is:
-
-<pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
-</pre>
-
-For Python 3 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 10 this is:
<pre>
$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
Note:
-The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 10 this is:
<pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
-</pre>
-
-For Python 3 this is:
-
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev python3-llfuse
</pre>
h3. Usage
target.save_new(name=target_name, owner_uuid=target_project)
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
+
+h2. Copy files from a collection another collection
+
+{% codeblock as python %}
+import arvados.collection
+
+source_collection = "x1u39-4zz18-krzg64ufvehgitl"
+target_collection = "x1u39-4zz18-67q94einb8ptznm"
+files_to_copy = ["folder1/sample1/sample1_R1.fastq",
+ "folder1/sample2/sample2_R1.fastq"]
+
+source = arvados.collection.CollectionReader(source_collection)
+target = arvados.collection.Collection(target_collection)
+
+for f in files_to_copy:
+ target.copy(f, "", source_collection=source)
+
+target.save()
+{% endcodeblock %}
+
+h2. Listing records with paging
+
+Use the @arvados.util.keyset_list_all@ helper method to iterate over all the records matching an optional filter. This method handles paging internally and returns results incrementally using a Python iterator. The first parameter of the method takes a @list@ method of an Arvados resource (@collections@, @container_requests@, etc).
+
+{% codeblock as python %}
+import arvados.util
+
+api = arvados.api()
+for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
+ print("got collection " + c["uuid"])
+{% endcodeblock %}
Note:
-The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 9 this is:
+The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 10 this is:
<pre>
-$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev
-</pre>
-
-For Python 3 this is
-
-<pre>
-$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl-dev
</pre>
If your version of @pip@ is 1.4 or newer, the @pip install@ command might give an error: "Could not find a version that satisfies the requirement arvados-python-client". If this happens, try @pip install --pre arvados-python-client@.
# "Install Ruby":../../install/ruby.html
-The SDK uses @curl@ which depends on the @libcurl@ C library. To build the module you may have to install additional packages. On Debian 9 this is:
+The SDK uses @curl@ which depends on the @libcurl@ C library. To build the module you may have to install additional packages. On Debian 10 this is:
<pre>
$ apt-get install build-essential libcurl4-openssl-dev
h3. 7. Set Docker image, base command, and input port for "sort" tool
-The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:9@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
+The "Docker Repository" is the name:tag of a "Docker image uploaded Arvados.":{{site.baseurl}}/user/topics/arv-docker.html (Use @arv-keepdocker --pull debian:10@) You can also find prepackaged bioinformatics tools on various sites, such as http://dockstore.org and http://biocontainers.pro/ .
!(screenshot)c6.png!
<pre>
requirements:
DockerRequirement:
- dockerPull: "debian:9"
+ dockerPull: "debian:10"
arv:dockerCollectionPDH: "feaf1fc916103d7cdab6489e1f8c3a2b+174"
</pre>
notextile. <pre>~/tutorials$ <code class="userinput">nano hello.cwl</code></pre>
-<notextile> {% code 'tutorial_hello_cwl' as yaml %} </notextile>
+<notextile> {% code tutorial_hello_cwl as yaml %} </notextile>
Next, add the file to the git repository. This tells @git@ that the file should be included on the next commit.
** Contains about resource consumption (RAM, cpu, disk, network) on the node while it was running
This is different from the log crunchstat.txt because it includes resource consumption of Arvados components that run on the node outside the container such as crunch-run and other processes related to the Keep file system.
-For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “mastermind” behind tracking which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then sending out the related container requests. Each step then has their own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
+For the highest level logs, the logs are tracking the container that ran the @arvados-cwl-runner@ process which you can think of as the “workflow runner”. It tracks which parts of the CWL workflow need to be run when, which have been run already, what order they need to be run, which can be run simultaneously, and so forth and then creates the necessary container requests. Each step has its own logs related to containers running a CWL step of the workflow including a log of standard error that contains the standard error of the code run in that CWL step. Those logs can be found by expanding the steps and clicking on the link to the log collection.
-Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “mastermind” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
+Let’s take a peek at a few of these logs to get you more familiar with them. First, we can look at the @stderr.txt@ of the highest level process. Again recall this should be of the “workflow runner” @arvados-cwl-runner@ process. You can click on the log to download it to your local machine, and when you look at the contents - you should see something like the following...
<pre><code>2020-06-22T20:30:04.737703197Z INFO /usr/bin/arvados-cwl-runner 2.0.3, arvados-python-client 2.0.3, cwltool 1.0.20190831161204
2020-06-22T20:30:04.743250012Z INFO Resolved '/var/lib/cwl/workflow.json#main' to 'file:///var/lib/cwl/workflow.json#main'
Liquid::Tag.instance_method(:initialize).bind(self).call(tag_name, markup, tokens)
if markup =~ Syntax
- @template_name = $1
+ @template_name_expr = $1
@language = $3
@attributes = {}
else
def render(context)
require 'coderay'
- partial = load_cached_partial(context)
+ partial = load_cached_partial(@template_name_expr, context)
html = ''
+ # be explicit about errors
+ context.exception_renderer = lambda do |exc|
+ exc.is_a?(Liquid::InternalError) ? "Liquid error: #{exc.cause.message}" : exc
+ end
+
context.stack do
html = CodeRay.scan(partial.root.nodelist.join, @language).div
end
partial = partial[1..-1]
end
+ # be explicit about errors
+ context.exception_renderer = lambda do |exc|
+ exc.is_a?(Liquid::InternalError) ? "Liquid error: #{exc.cause.message}" : exc
+ end
+
context.stack do
html = CodeRay.scan(partial, @language).div
end
#
# SPDX-License-Identifier: Apache-2.0
-# Based on Debian Stretch
+# Based on Debian
FROM debian:buster-slim
MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
# apt.arvados.org
-deb http://apt.arvados.org/ buster-dev main
+deb http://apt.arvados.org/buster buster-dev main
# apt.arvados.org
-deb http://apt.arvados.org/ buster main
+deb http://apt.arvados.org/buster buster main
# apt.arvados.org
-deb http://apt.arvados.org/ buster-testing main
+deb http://apt.arvados.org/buster buster-testing main
VOLUME /var/lib/docker
RUN mkdir -p /etc/apt/sources.list.d && \
- echo deb http://apt.arvados.org/ jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
+ echo deb http://apt.arvados.org/jessie jessie main > /etc/apt/sources.list.d/apt.arvados.org.list && \
apt-get clean && \
apt-get update && \
apt-get install -yq --no-install-recommends -o Acquire::Retries=6 \
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): %s", err)
}
- postgresUid, err := strconv.Atoi(postgresUser.Uid)
+ postgresUID, err := strconv.Atoi(postgresUser.Uid)
if err != nil {
return fmt.Errorf("user.Lookup(\"postgres\"): non-numeric uid?: %q", postgresUser.Uid)
}
if err != nil {
return err
}
- err = os.Chown(datadir, postgresUid, 0)
+ err = os.Chown(datadir, postgresUID, 0)
if err != nil {
return err
}
return prog
}
-// Run prog with args, using dir as working directory. If ctx is
-// cancelled while the child is running, RunProgram terminates the
-// child, waits for it to exit, then returns.
+// RunProgram runs prog with args, using dir as working directory. If ctx is
+// cancelled while the child is running, RunProgram terminates the child, waits
+// for it to exit, then returns.
//
// Child's environment will have our env vars, plus any given in env.
//
if cluster.ManagementToken == "" {
cluster.ManagementToken = randomHexString(64)
}
- if cluster.API.RailsSessionSecretToken == "" {
- cluster.API.RailsSessionSecretToken = randomHexString(64)
- }
if cluster.Collections.BlobSigningKey == "" {
cluster.Collections.BlobSigningKey = randomHexString(64)
}
sha1pkix := sha1.Sum([]byte(pkix))
md5fp = ""
sha1fp = ""
- for i := 0; i < len(md5pkix); i += 1 {
+ for i := 0; i < len(md5pkix); i++ {
md5fp += fmt.Sprintf(":%02x", md5pkix[i])
}
- for i := 0; i < len(sha1pkix); i += 1 {
+ for i := 0; i < len(sha1pkix); i++ {
sha1fp += fmt.Sprintf(":%02x", sha1pkix[i])
}
return md5fp[1:], sha1fp[1:], nil
}
}
-func (az *ec2InstanceSet) Stop() {
+func (instanceSet *ec2InstanceSet) Stop() {
}
type ec2Instance struct {
//
// SPDX-License-Identifier: Apache-2.0
-// package cmd helps define reusable functions that can be exposed as
+// Package cmd helps define reusable functions that can be exposed as
// [subcommands of] command line programs.
package cmd
in := `
Clusters:
z1234:
- ManagementToken: xyzzy
- SystemRootToken: xyzzy
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
API:
MaxItemsPerResponse: 1234
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
PostgreSQL:
Connection:
sslmode: require
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
# serving a single incoming multi-cluster (federated) request.
MaxRequestAmplification: 4
- # RailsSessionSecretToken is a string of alphanumeric characters
- # used by Rails to sign session tokens. IMPORTANT: This is a
- # site secret. It should be at least 50 characters.
- RailsSessionSecretToken: ""
-
# Maximum wall clock time to spend handling an incoming request.
RequestTimeout: 5m
}
`)
cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
}
`)
cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
// The resulting ManagementToken should be the one set up on the test server.
c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
}
content := []byte(fmtKeepproxyConfig("", true))
cluster, err := testLoadLegacyConfig(content, f, c)
- c.Check(err, check.IsNil)
- c.Check(cluster, check.NotNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(cluster, check.NotNil)
c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
f := "-legacy-git-httpd-config"
cluster, err := testLoadLegacyConfig(content, f, c)
- c.Check(err, check.IsNil)
- c.Check(cluster, check.NotNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(cluster, check.NotNil)
c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
}
`)
cluster, err := testLoadLegacyConfig(content, "-legacy-git-httpd-config", c)
- c.Check(err, check.IsNil)
+ c.Assert(err, check.IsNil)
// The resulting ManagementToken should be the one set up on the test server.
c.Check(cluster.ManagementToken, check.Equals, TestServerManagementToken)
}
content := []byte(fmtKeepBalanceConfig(""))
cluster, err := testLoadLegacyConfig(content, f, c)
- c.Check(err, check.IsNil)
- c.Check(cluster, check.NotNil)
+ c.Assert(err, check.IsNil)
+ c.Assert(cluster, check.NotNil)
c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
c.Check(cluster.Services.Keepbalance.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
c.Check(cluster.Collections.BalanceCollectionBuffers, check.Equals, 1000)
"API.MaxKeepBlobBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
- "API.RailsSessionSecretToken": false,
"API.RequestTimeout": true,
"API.SendTimeout": true,
"API.WebsocketClientEventQueue": false,
Clusters:
xxxxx:
+ # Token used internally by Arvados components to authenticate to
+ # one another. Use a string of at least 50 random alphanumerics.
SystemRootToken: ""
# Token to be included in all healthcheck requests. Disabled by default.
# serving a single incoming multi-cluster (federated) request.
MaxRequestAmplification: 4
- # RailsSessionSecretToken is a string of alphanumeric characters
- # used by Rails to sign session tokens. IMPORTANT: This is a
- # site secret. It should be at least 50 characters.
- RailsSessionSecretToken: ""
-
# Maximum wall clock time to spend handling an incoming request.
RequestTimeout: 5m
"io"
"io/ioutil"
"os"
+ "regexp"
"strings"
"git.arvados.org/arvados.git/sdk/go/arvados"
// Check for known mistakes
for id, cc := range cfg.Clusters {
for _, err = range []error{
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken),
+ ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey),
checkKeyConflict(fmt.Sprintf("Clusters.%s.PostgreSQL.Connection", id), cc.PostgreSQL.Connection),
ldr.checkEmptyKeepstores(cc),
ldr.checkUnlistedKeepstores(cc),
return &cfg, nil
}
+var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`)
+var acceptableTokenLength = 32
+
+func (ldr *Loader) checkToken(label, token string) error {
+ if token == "" {
+ ldr.Logger.Warnf("%s: secret token is not set (use %d+ random characters from a-z, A-Z, 0-9)", label, acceptableTokenLength)
+ } else if !acceptableTokenRe.MatchString(token) {
+ return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label)
+ } else if len(token) < acceptableTokenLength {
+ ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength)
+ }
+ return nil
+}
+
func checkKeyConflict(label string, m map[string]string) error {
saw := map[string]bool{}
for k := range m {
_, err := testLoader(c, `
Clusters:
zzzzz:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
postgresql: {}
BadKey: {}
Containers: {}
err = yaml.Unmarshal(buf, &loaded)
c.Assert(err, check.IsNil)
+ c.Check(logbuf.String(), check.Matches, `(?ms).*SystemRootToken: secret token is not set.*`)
+ c.Check(logbuf.String(), check.Matches, `(?ms).*ManagementToken: secret token is not set.*`)
+ c.Check(logbuf.String(), check.Matches, `(?ms).*Collections.BlobSigningKey: secret token is not set.*`)
+ logbuf.Reset()
loader.logExtraKeys(loaded, supplied, "")
c.Check(logbuf.String(), check.Equals, "")
}
var logbuf bytes.Buffer
logger := logrus.New()
logger.Out = &logbuf
- cfg, err := testLoader(c, `{"Clusters":{"zzzzz":{}}}`, &logbuf).Load()
+ cfg, err := testLoader(c, `
+Clusters:
+ zzzzz:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa`, &logbuf).Load()
c.Assert(err, check.IsNil)
yaml, err := yaml.Marshal(cfg)
c.Assert(err, check.IsNil)
c.Check(logbuf.String(), check.Equals, "")
}
+func (s *LoadSuite) TestUnacceptableTokens(c *check.C) {
+ for _, trial := range []struct {
+ short bool
+ configPath string
+ example string
+ }{
+ {false, "SystemRootToken", "SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa_b_c"},
+ {false, "ManagementToken", "ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa b c"},
+ {false, "ManagementToken", "ManagementToken: \"$aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabc\""},
+ {false, "Collections.BlobSigningKey", "Collections: {BlobSigningKey: \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa⛵\"}"},
+ {true, "SystemRootToken", "SystemRootToken: a_b_c"},
+ {true, "ManagementToken", "ManagementToken: a b c"},
+ {true, "ManagementToken", "ManagementToken: \"$abc\""},
+ {true, "Collections.BlobSigningKey", "Collections: {BlobSigningKey: \"⛵\"}"},
+ } {
+ c.Logf("trying bogus config: %s", trial.example)
+ _, err := testLoader(c, "Clusters:\n zzzzz:\n "+trial.example, nil).Load()
+ if trial.short {
+ c.Check(err, check.ErrorMatches, `Clusters.zzzzz.`+trial.configPath+`: unacceptable characters in token.*`)
+ } else {
+ c.Check(err, check.ErrorMatches, `Clusters.zzzzz.`+trial.configPath+`: unacceptable characters in token.*`)
+ }
+ }
+}
+
func (s *LoadSuite) TestPostgreSQLKeyConflict(c *check.C) {
_, err := testLoader(c, `
Clusters:
// it to the router package would cause a circular dependency
// router->arvadostest->ctrlctx->router.)
type RoutableFunc func(ctx context.Context, opts interface{}) (interface{}, error)
+
+type RoutableFuncWrapper func(RoutableFunc) RoutableFunc
+
+// ComposeWrappers (w1, w2, w3, ...) returns a RoutableFuncWrapper that
+// composes w1, w2, w3, ... such that w1 is the outermost wrapper.
+func ComposeWrappers(wraps ...RoutableFuncWrapper) RoutableFuncWrapper {
+ return func(f RoutableFunc) RoutableFunc {
+ for i := len(wraps) - 1; i >= 0; i-- {
+ f = wraps[i](f)
+ }
+ return f
+ }
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package controller
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "git.arvados.org/arvados.git/sdk/go/httpserver"
+ "github.com/sirupsen/logrus"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+var _ = check.Suite(&AuthSuite{})
+
+type AuthSuite struct {
+ log logrus.FieldLogger
+ // testServer and testHandler are the controller being tested,
+ // "zhome".
+ testServer *httpserver.Server
+ testHandler *Handler
+ // remoteServer ("zzzzz") forwards requests to the Rails API
+ // provided by the integration test environment.
+ remoteServer *httpserver.Server
+ // remoteMock ("zmock") appends each incoming request to
+ // remoteMockRequests, and returns 200 with an empty JSON
+ // object.
+ remoteMock *httpserver.Server
+ remoteMockRequests []http.Request
+
+ fakeProvider *arvadostest.OIDCProvider
+}
+
+func (s *AuthSuite) SetUpTest(c *check.C) {
+ s.log = ctxlog.TestLogger(c)
+
+ s.remoteServer = newServerFromIntegrationTestEnv(c)
+ c.Assert(s.remoteServer.Start(), check.IsNil)
+
+ s.remoteMock = newServerFromIntegrationTestEnv(c)
+ s.remoteMock.Server.Handler = http.HandlerFunc(http.NotFound)
+ c.Assert(s.remoteMock.Start(), check.IsNil)
+
+ s.fakeProvider = arvadostest.NewOIDCProvider(c)
+ s.fakeProvider.AuthEmail = "active-user@arvados.local"
+ s.fakeProvider.AuthEmailVerified = true
+ s.fakeProvider.AuthName = "Fake User Name"
+ s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{}
+ s.fakeProvider.ValidClientID = "test%client$id"
+ s.fakeProvider.ValidClientSecret = "test#client/secret"
+
+ cluster := &arvados.Cluster{
+ ClusterID: "zhome",
+ PostgreSQL: integrationTestCluster().PostgreSQL,
+ ForceLegacyAPI14: forceLegacyAPI14,
+ SystemRootToken: arvadostest.SystemRootToken,
+ }
+ cluster.TLS.Insecure = true
+ cluster.API.MaxItemsPerResponse = 1000
+ cluster.API.MaxRequestAmplification = 4
+ cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+ arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+ arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost/")
+
+ cluster.RemoteClusters = map[string]arvados.RemoteCluster{
+ "zzzzz": {
+ Host: s.remoteServer.Addr,
+ Proxy: true,
+ Scheme: "http",
+ },
+ "zmock": {
+ Host: s.remoteMock.Addr,
+ Proxy: true,
+ Scheme: "http",
+ },
+ "*": {
+ Scheme: "https",
+ },
+ }
+ cluster.Login.OpenIDConnect.Enable = true
+ cluster.Login.OpenIDConnect.Issuer = s.fakeProvider.Issuer.URL
+ cluster.Login.OpenIDConnect.ClientID = s.fakeProvider.ValidClientID
+ cluster.Login.OpenIDConnect.ClientSecret = s.fakeProvider.ValidClientSecret
+ cluster.Login.OpenIDConnect.EmailClaim = "email"
+ cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+
+ s.testHandler = &Handler{Cluster: cluster}
+ s.testServer = newServerFromIntegrationTestEnv(c)
+ s.testServer.Server.Handler = httpserver.HandlerWithContext(
+ ctxlog.Context(context.Background(), s.log),
+ httpserver.AddRequestIDs(httpserver.LogRequests(s.testHandler)))
+ c.Assert(s.testServer.Start(), check.IsNil)
+}
+
+func (s *AuthSuite) TestLocalOIDCAccessToken(c *check.C) {
+ req := httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
+ req.Header.Set("Authorization", "Bearer "+s.fakeProvider.ValidAccessToken())
+ rr := httptest.NewRecorder()
+ s.testServer.Server.Handler.ServeHTTP(rr, req)
+ resp := rr.Result()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ var u arvados.User
+ c.Check(json.NewDecoder(resp.Body).Decode(&u), check.IsNil)
+ c.Check(u.UUID, check.Equals, arvadostest.ActiveUserUUID)
+ c.Check(u.OwnerUUID, check.Equals, "zzzzz-tpzed-000000000000000")
+
+ // Request again to exercise cache.
+ req = httptest.NewRequest("GET", "/arvados/v1/users/current", nil)
+ req.Header.Set("Authorization", "Bearer "+s.fakeProvider.ValidAccessToken())
+ rr = httptest.NewRecorder()
+ s.testServer.Server.Handler.ServeHTTP(rr, req)
+ resp = rr.Result()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+}
func fetchRemoteCollectionByUUID(
h *genericFederatedRequestHandler,
effectiveMethod string,
- clusterId *string,
+ clusterID *string,
uuid string,
remainder string,
w http.ResponseWriter,
if uuid != "" {
// Collection UUID GET request
- *clusterId = uuid[0:5]
- if *clusterId != "" && *clusterId != h.handler.Cluster.ClusterID {
+ *clusterID = uuid[0:5]
+ if *clusterID != "" && *clusterID != h.handler.Cluster.ClusterID {
// request for remote collection by uuid
- resp, err := h.handler.remoteClusterRequest(*clusterId, req)
- newResponse, err := rewriteSignatures(*clusterId, "", resp, err)
+ resp, err := h.handler.remoteClusterRequest(*clusterID, req)
+ newResponse, err := rewriteSignatures(*clusterID, "", resp, err)
h.handler.proxy.ForwardResponse(w, newResponse, err)
return true
}
func fetchRemoteCollectionByPDH(
h *genericFederatedRequestHandler,
effectiveMethod string,
- clusterId *string,
+ clusterID *string,
uuid string,
remainder string,
w http.ResponseWriter,
func remoteContainerRequestCreate(
h *genericFederatedRequestHandler,
effectiveMethod string,
- clusterId *string,
+ clusterID *string,
uuid string,
remainder string,
w http.ResponseWriter,
return true
}
- if *clusterId == "" || *clusterId == h.handler.Cluster.ClusterID {
+ if *clusterID == "" || *clusterID == h.handler.Cluster.ClusterID {
// Submitting container request to local cluster. No
// need to set a runtime_token (rails api will create
// one when the container runs) or do a remote cluster
crString, ok := request["container_request"].(string)
if ok {
- var crJson map[string]interface{}
- err := json.Unmarshal([]byte(crString), &crJson)
+ var crJSON map[string]interface{}
+ err := json.Unmarshal([]byte(crString), &crJSON)
if err != nil {
httpserver.Error(w, err.Error(), http.StatusBadRequest)
return true
}
- request["container_request"] = crJson
+ request["container_request"] = crJSON
}
containerRequest, ok := request["container_request"].(map[string]interface{})
req.ContentLength = int64(buf.Len())
req.Header.Set("Content-Length", fmt.Sprintf("%v", buf.Len()))
- resp, err := h.handler.remoteClusterRequest(*clusterId, req)
+ resp, err := h.handler.remoteClusterRequest(*clusterID, req)
h.handler.proxy.ForwardResponse(w, resp, err)
return true
}
type federatedRequestDelegate func(
h *genericFederatedRequestHandler,
effectiveMethod string,
- clusterId *string,
+ clusterID *string,
uuid string,
remainder string,
w http.ResponseWriter,
clusterID string, uuids []string) (rp []map[string]interface{}, kind string, err error) {
found := make(map[string]bool)
- prev_len_uuids := len(uuids) + 1
+ prevLenUuids := len(uuids) + 1
// Loop while
// (1) there are more uuids to query
// (2) we're making progress - on each iteration the set of
// uuids we are expecting for must shrink.
- for len(uuids) > 0 && len(uuids) < prev_len_uuids {
+ for len(uuids) > 0 && len(uuids) < prevLenUuids {
var remoteReq http.Request
remoteReq.Header = req.Header
remoteReq.Method = "POST"
l = append(l, u)
}
}
- prev_len_uuids = len(uuids)
+ prevLenUuids = len(uuids)
uuids = l
}
}
func (h *genericFederatedRequestHandler) handleMultiClusterQuery(w http.ResponseWriter,
- req *http.Request, clusterId *string) bool {
+ req *http.Request, clusterID *string) bool {
var filters [][]interface{}
err := json.Unmarshal([]byte(req.Form.Get("filters")), &filters)
if rhs, ok := filter[2].([]interface{}); ok {
for _, i := range rhs {
if u, ok := i.(string); ok && len(u) == 27 {
- *clusterId = u[0:5]
+ *clusterID = u[0:5]
queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u)
- expectCount += 1
+ expectCount++
}
}
}
} else if op == "=" {
if u, ok := filter[2].(string); ok && len(u) == 27 {
- *clusterId = u[0:5]
+ *clusterID = u[0:5]
queryClusters[u[0:5]] = append(queryClusters[u[0:5]], u)
- expectCount += 1
+ expectCount++
}
} else {
return false
func (h *genericFederatedRequestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
m := h.matcher.FindStringSubmatch(req.URL.Path)
- clusterId := ""
+ clusterID := ""
if len(m) > 0 && m[2] != "" {
- clusterId = m[2]
+ clusterID = m[2]
}
// Get form parameters from URL and form body (if POST).
// Check if the parameters have an explicit cluster_id
if req.Form.Get("cluster_id") != "" {
- clusterId = req.Form.Get("cluster_id")
+ clusterID = req.Form.Get("cluster_id")
}
// Handle the POST-as-GET special case (workaround for large
}
if effectiveMethod == "GET" &&
- clusterId == "" &&
+ clusterID == "" &&
req.Form.Get("filters") != "" &&
- h.handleMultiClusterQuery(w, req, &clusterId) {
+ h.handleMultiClusterQuery(w, req, &clusterID) {
return
}
uuid = m[1][1:]
}
for _, d := range h.delegates {
- if d(h, effectiveMethod, &clusterId, uuid, m[3], w, req) {
+ if d(h, effectiveMethod, &clusterID, uuid, m[3], w, req) {
return
}
}
- if clusterId == "" || clusterId == h.handler.Cluster.ClusterID {
+ if clusterID == "" || clusterID == h.handler.Cluster.ClusterID {
h.next.ServeHTTP(w, req)
} else {
- resp, err := h.handler.remoteClusterRequest(clusterId, req)
+ resp, err := h.handler.remoteClusterRequest(clusterID, req)
h.handler.proxy.ForwardResponse(w, resp, err)
}
}
return updatedReq, nil
}
+ ctxlog.FromContext(req.Context()).Infof("saltAuthToken: cluster %s token %s remote %s", h.Cluster.ClusterID, creds.Tokens[0], remote)
token, err := auth.SaltToken(creds.Tokens[0], remote)
if err == auth.ErrObsoleteToken {
- // If the token exists in our own database, salt it
- // for the remote. Otherwise, assume it was issued by
- // the remote, and pass it through unmodified.
+ // If the token exists in our own database for our own
+ // user, salt it for the remote. Otherwise, assume it
+ // was issued by the remote, and pass it through
+ // unmodified.
currentUser, ok, err := h.validateAPItoken(req, creds.Tokens[0])
if err != nil {
return nil, err
- } else if !ok {
- // Not ours; pass through unmodified.
+ } else if !ok || strings.HasPrefix(currentUser.UUID, remote) {
+ // Unknown, or cached + belongs to remote;
+ // pass through unmodified.
token = creds.Tokens[0]
} else {
// Found; make V2 version and salt it.
} else if err != nil {
return nil, err
}
+ if strings.HasPrefix(aca.UUID, remoteID) {
+ // We have it cached here, but
+ // the token belongs to the
+ // remote target itself, so
+ // pass it through unmodified.
+ tokens = append(tokens, token)
+ continue
+ }
salted, err := auth.SaltToken(aca.TokenV2(), remoteID)
if err != nil {
return nil, err
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- return conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ if err != nil {
+ return resp, err
+ }
+ if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
+ // Copy the updated user record to the local cluster
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ if err != nil {
+ return arvados.User{}, err
+ }
+ }
+ return resp, err
}
func (conn *Conn) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) {
w.WriteHeader(200)
w.Write([]byte(`{"kind": "arvados#containerList", "items": [{"uuid": "zhome-xvhdp-cr6queuedcontnr", "command": ["efg"]}]}`))
}
- callCount += 1
+ callCount++
})).Close()
req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
w.WriteHeader(200)
w.Write([]byte(`{"kind": "arvados#containerList", "items": []}`))
}
- callCount += 1
+ callCount++
})).Close()
req := httptest.NewRequest("GET", fmt.Sprintf("/arvados/v1/containers?count=none&filters=%s",
url.QueryEscape(fmt.Sprintf(`[["uuid", "in", ["%v", "zhome-xvhdp-cr5queuedcontnr", "zhome-xvhdp-cr6queuedcontnr"]]]`,
"sync"
"time"
+ "git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/lib/controller/federation"
+ "git.arvados.org/arvados.git/lib/controller/localdb"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/router"
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/health"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
Routes: health.Routes{"ping": func() error { _, err := h.db(context.TODO()); return err }},
})
- rtr := router.New(federation.New(h.Cluster), ctrlctx.WrapCallsInTransactions(h.db))
+ oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.db)
+ rtr := router.New(federation.New(h.Cluster), api.ComposeWrappers(ctrlctx.WrapCallsInTransactions(h.db), oidcAuthorizer.WrapCalls))
mux.Handle("/arvados/v1/config", rtr)
mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr)
hs := http.NotFoundHandler()
hs = prepend(hs, h.proxyRailsAPI)
hs = h.setupProxyRemoteCluster(hs)
+ hs = prepend(hs, oidcAuthorizer.Middleware)
mux.Handle("/", hs)
h.handlerStack = mux
"bytes"
"context"
"encoding/json"
+ "fmt"
"io"
+ "io/ioutil"
"math"
"net"
"net/http"
"net/url"
"os"
+ "os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"git.arvados.org/arvados.git/lib/boot"
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/service"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
type IntegrationSuite struct {
testClusters map[string]*testCluster
+ oidcprovider *arvadostest.OIDCProvider
}
func (s *IntegrationSuite) SetUpSuite(c *check.C) {
}
cwd, _ := os.Getwd()
+
+ s.oidcprovider = arvadostest.NewOIDCProvider(c)
+ s.oidcprovider.AuthEmail = "user@example.com"
+ s.oidcprovider.AuthEmailVerified = true
+ s.oidcprovider.AuthName = "Example User"
+ s.oidcprovider.ValidClientID = "clientid"
+ s.oidcprovider.ValidClientSecret = "clientsecret"
+
s.testClusters = map[string]*testCluster{
"z1111": nil,
"z2222": nil,
ActivateUsers: true
`
}
+ if id == "z1111" {
+ yaml += `
+ Login:
+ LoginCluster: z1111
+ OpenIDConnect:
+ Enable: true
+ Issuer: ` + s.oidcprovider.Issuer.URL + `
+ ClientID: ` + s.oidcprovider.ValidClientID + `
+ ClientSecret: ` + s.oidcprovider.ValidClientSecret + `
+ EmailClaim: email
+ EmailVerifiedClaim: email_verified
+`
+ } else {
+ yaml += `
+ Login:
+ LoginCluster: z1111
+`
+ }
loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
loader.Path = "-"
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestS3WithFederatedToken(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not in PATH")
+ return
+ }
+
+ testText := "IntegrationSuite.TestS3WithFederatedToken"
+
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ userctx1, ac1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+ conn3 := s.conn("z3333")
+
+ createColl := func(clusterID string) arvados.Collection {
+ _, ac, kc := s.clientsWithToken(clusterID, ac1.AuthToken)
+ var coll arvados.Collection
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, testText)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = s.conn(clusterID).CollectionCreate(userctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ return coll
+ }
+
+ for _, trial := range []struct {
+ clusterID string // create the collection on this cluster (then use z3333 to access it)
+ token string
+ }{
+ // Try the hardest test first: z3333 hasn't seen
+ // z1111's token yet, and we're just passing the
+ // opaque secret part, so z3333 has to guess that it
+ // belongs to z1111.
+ {"z1111", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z3333", strings.Split(ac1.AuthToken, "/")[2]},
+ {"z1111", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ {"z3333", strings.Replace(ac1.AuthToken, "/", "_", -1)},
+ } {
+ c.Logf("================ %v", trial)
+ coll := createColl(trial.clusterID)
+
+ cfgjson, err := conn3.ConfigGet(userctx1)
+ c.Assert(err, check.IsNil)
+ var cluster arvados.Cluster
+ err = json.Unmarshal(cfgjson, &cluster)
+ c.Assert(err, check.IsNil)
+
+ c.Logf("TokenV2 is %s", ac1.AuthToken)
+ host := cluster.Services.WebDAV.ExternalURL.Host
+ s3args := []string{
+ "--ssl", "--no-check-certificate",
+ "--host=" + host, "--host-bucket=" + host,
+ "--access_key=" + trial.token, "--secret_key=" + trial.token,
+ }
+ buf, err := exec.Command("s3cmd", append(s3args, "ls", "s3://"+coll.UUID)...).CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* `+fmt.Sprintf("%d", len(testText))+` +s3://`+coll.UUID+`/test.txt\n`)
+
+ buf, _ = exec.Command("s3cmd", append(s3args, "get", "s3://"+coll.UUID+"/test.txt", c.MkDir()+"/tmpfile")...).CombinedOutput()
+ // Command fails because we don't return Etag header.
+ flen := strconv.Itoa(len(testText))
+ c.Check(string(buf), check.Matches, `(?ms).*`+flen+` (bytes in|of `+flen+`).*`)
+ }
+}
+
func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
conn1 := s.conn("z1111")
conn3 := s.conn("z3333")
c.Check(len(outLinks.Items), check.Equals, 1)
}
+
+func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
+ conn1 := s.conn("z1111")
+ rootctx1, _, _ := s.rootClients("z1111")
+ s.userClients(rootctx1, c, conn1, "z1111", true)
+
+ accesstoken := s.oidcprovider.ValidAccessToken()
+
+ for _, clusterid := range []string{"z1111", "z2222"} {
+ c.Logf("trying clusterid %s", clusterid)
+
+ conn := s.conn(clusterid)
+ ctx, ac, kc := s.clientsWithToken(clusterid, accesstoken)
+
+ var coll arvados.Collection
+
+ // Write some file data and create a collection
+ {
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, "IntegrationSuite.TestOIDCAccessTokenAuth")
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ coll, err = conn.CollectionCreate(ctx, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+ }
+
+ // Read the collection & file data
+ {
+ user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ c.Assert(err, check.IsNil)
+ c.Check(user.FullName, check.Equals, "Example User")
+ coll, err = conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
+ c.Assert(err, check.IsNil)
+ c.Check(coll.ManifestText, check.Not(check.Equals), "")
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.Open("test.txt")
+ c.Assert(err, check.IsNil)
+ buf, err := ioutil.ReadAll(f)
+ c.Assert(err, check.IsNil)
+ c.Check(buf, check.DeepEquals, []byte("IntegrationSuite.TestOIDCAccessTokenAuth"))
+ }
+ }
+}
"context"
"crypto/hmac"
"crypto/sha256"
+ "database/sql"
"encoding/base64"
"errors"
"fmt"
+ "io"
"net/http"
"net/url"
"strings"
"text/template"
"time"
+ "git.arvados.org/arvados.git/lib/controller/api"
+ "git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/coreos/go-oidc"
+ lru "github.com/hashicorp/golang-lru"
+ "github.com/jmoiron/sqlx"
+ "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"google.golang.org/api/option"
"google.golang.org/api/people/v1"
)
+const (
+ tokenCacheSize = 1000
+ tokenCacheNegativeTTL = time.Minute * 5
+ tokenCacheTTL = time.Minute * 10
+)
+
type oidcLoginController struct {
Cluster *arvados.Cluster
RailsProxy *railsProxy
return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest)
}
+// claimser can decode arbitrary claims into a map. Implemented by
+// *oauth2.IDToken and *oauth2.UserInfo.
+type claimser interface {
+ Claims(interface{}) error
+}
+
// Use a person's token to get all of their email addresses, with the
// primary address at index 0. The provided defaultAddr is always
// included in the returned slice, and is used as the primary if the
// Google API does not indicate one.
-func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
+func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, claimser claimser) (*rpc.UserSessionAuthInfo, error) {
var ret rpc.UserSessionAuthInfo
defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned")
var claims map[string]interface{}
- if err := idToken.Claims(&claims); err != nil {
- return nil, fmt.Errorf("error extracting claims from ID token: %s", err)
+ if err := claimser.Claims(&claims); err != nil {
+ return nil, fmt.Errorf("error extracting claims from token: %s", err)
} else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" {
// Fall back to this info if the People API call
// (below) doesn't return a primary && verified email.
fmt.Fprintf(mac, "%x %s %s", s.Time, s.Remote, s.ReturnTo)
return mac.Sum(nil)
}
+
+func OIDCAccessTokenAuthorizer(cluster *arvados.Cluster, getdb func(context.Context) (*sqlx.DB, error)) *oidcTokenAuthorizer {
+ // We want ctrl to be nil if the chosen controller is not a
+ // *oidcLoginController, so we can ignore the 2nd return value
+ // of this type cast.
+ ctrl, _ := chooseLoginController(cluster, railsproxy.NewConn(cluster)).(*oidcLoginController)
+ cache, err := lru.New2Q(tokenCacheSize)
+ if err != nil {
+ panic(err)
+ }
+ return &oidcTokenAuthorizer{
+ ctrl: ctrl,
+ getdb: getdb,
+ cache: cache,
+ }
+}
+
+type oidcTokenAuthorizer struct {
+ ctrl *oidcLoginController
+ getdb func(context.Context) (*sqlx.DB, error)
+ cache *lru.TwoQueueCache
+}
+
+func (ta *oidcTokenAuthorizer) Middleware(w http.ResponseWriter, r *http.Request, next http.Handler) {
+ if ta.ctrl == nil {
+ // Not using a compatible (OIDC) login controller.
+ } else if authhdr := strings.Split(r.Header.Get("Authorization"), " "); len(authhdr) > 1 && (authhdr[0] == "OAuth2" || authhdr[0] == "Bearer") {
+ err := ta.registerToken(r.Context(), authhdr[1])
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ }
+ next.ServeHTTP(w, r)
+}
+
+func (ta *oidcTokenAuthorizer) WrapCalls(origFunc api.RoutableFunc) api.RoutableFunc {
+ if ta.ctrl == nil {
+ // Not using a compatible (OIDC) login controller.
+ return origFunc
+ }
+ return func(ctx context.Context, opts interface{}) (_ interface{}, err error) {
+ creds, ok := auth.FromContext(ctx)
+ if !ok {
+ return origFunc(ctx, opts)
+ }
+ // Check each token in the incoming request. If any
+ // are OAuth2 access tokens, swap them out for Arvados
+ // tokens.
+ for _, tok := range creds.Tokens {
+ err = ta.registerToken(ctx, tok)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return origFunc(ctx, opts)
+ }
+}
+
+// registerToken checks whether tok is a valid OIDC Access Token and,
+// if so, ensures that an api_client_authorizations row exists so that
+// RailsAPI will accept it as an Arvados token.
+func (ta *oidcTokenAuthorizer) registerToken(ctx context.Context, tok string) error {
+ if tok == ta.ctrl.Cluster.SystemRootToken || strings.HasPrefix(tok, "v2/") {
+ return nil
+ }
+ if cached, hit := ta.cache.Get(tok); !hit {
+ // Fall through to database and OIDC provider checks
+ // below
+ } else if exp, ok := cached.(time.Time); ok {
+ // cached negative result (value is expiry time)
+ if time.Now().Before(exp) {
+ return nil
+ }
+ ta.cache.Remove(tok)
+ } else {
+ // cached positive result
+ aca := cached.(arvados.APIClientAuthorization)
+ var expiring bool
+ if aca.ExpiresAt != "" {
+ t, err := time.Parse(time.RFC3339Nano, aca.ExpiresAt)
+ if err != nil {
+ return fmt.Errorf("error parsing expires_at value: %w", err)
+ }
+ expiring = t.Before(time.Now().Add(time.Minute))
+ }
+ if !expiring {
+ return nil
+ }
+ }
+
+ db, err := ta.getdb(ctx)
+ if err != nil {
+ return err
+ }
+ tx, err := db.Beginx()
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+ ctx = ctrlctx.NewWithTransaction(ctx, tx)
+
+ // We use hmac-sha256(accesstoken,systemroottoken) as the
+ // secret part of our own token, and avoid storing the auth
+ // provider's real secret in our database.
+ mac := hmac.New(sha256.New, []byte(ta.ctrl.Cluster.SystemRootToken))
+ io.WriteString(mac, tok)
+ hmac := fmt.Sprintf("%x", mac.Sum(nil))
+
+ var expiring bool
+ err = tx.QueryRowContext(ctx, `select (expires_at is not null and expires_at - interval '1 minute' <= current_timestamp at time zone 'UTC') from api_client_authorizations where api_token=$1`, hmac).Scan(&expiring)
+ if err != nil && err != sql.ErrNoRows {
+ return fmt.Errorf("database error while checking token: %w", err)
+ } else if err == nil && !expiring {
+ // Token is already in the database as an Arvados
+ // token, and isn't about to expire, so we can pass it
+ // through to RailsAPI etc. regardless of whether it's
+ // an OIDC access token.
+ return nil
+ }
+ updating := err == nil
+
+ // Check whether the token is a valid OIDC access token. If
+ // so, swap it out for an Arvados token (creating/updating an
+ // api_client_authorizations row if needed) which downstream
+ // server components will accept.
+ err = ta.ctrl.setup()
+ if err != nil {
+ return fmt.Errorf("error setting up OpenID Connect provider: %s", err)
+ }
+ oauth2Token := &oauth2.Token{
+ AccessToken: tok,
+ }
+ userinfo, err := ta.ctrl.provider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token))
+ if err != nil {
+ ta.cache.Add(tok, time.Now().Add(tokenCacheNegativeTTL))
+ return nil
+ }
+ ctxlog.FromContext(ctx).WithField("userinfo", userinfo).Debug("(*oidcTokenAuthorizer)registerToken: got userinfo")
+ authinfo, err := ta.ctrl.getAuthInfo(ctx, oauth2Token, userinfo)
+ if err != nil {
+ return err
+ }
+
+ // Expiry time for our token is one minute longer than our
+ // cache TTL, so we don't pass it through to RailsAPI just as
+ // it's expiring.
+ exp := time.Now().UTC().Add(tokenCacheTTL + time.Minute)
+
+ var aca arvados.APIClientAuthorization
+ if updating {
+ _, err = tx.ExecContext(ctx, `update api_client_authorizations set expires_at=$1 where api_token=$2`, exp, hmac)
+ if err != nil {
+ return fmt.Errorf("error updating token expiry time: %w", err)
+ }
+ ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: updated api_client_authorizations row")
+ } else {
+ aca, err = createAPIClientAuthorization(ctx, ta.ctrl.RailsProxy, ta.ctrl.Cluster.SystemRootToken, *authinfo)
+ if err != nil {
+ return err
+ }
+ _, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID)
+ if err != nil {
+ return fmt.Errorf("error adding OIDC access token to database: %w", err)
+ }
+ aca.APIToken = hmac
+ ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row")
+ }
+ err = tx.Commit()
+ if err != nil {
+ return err
+ }
+ ta.cache.Add(tok, aca)
+ return nil
+}
import (
"bytes"
"context"
- "crypto/rand"
- "crypto/rsa"
- "encoding/base64"
"encoding/json"
"fmt"
"net/http"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
- jose "gopkg.in/square/go-jose.v2"
)
// Gocheck boilerplate
var _ = check.Suite(&OIDCLoginSuite{})
type OIDCLoginSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
- fakeIssuer *httptest.Server
- fakePeopleAPI *httptest.Server
- fakePeopleAPIResponse map[string]interface{}
- issuerKey *rsa.PrivateKey
-
- // expected token request
- validCode string
- validClientID string
- validClientSecret string
- // desired response from token endpoint
- authEmail string
- authEmailVerified bool
- authName string
+ cluster *arvados.Cluster
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+ fakeProvider *arvadostest.OIDCProvider
}
func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
}
func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
- var err error
- s.issuerKey, err = rsa.GenerateKey(rand.Reader, 2048)
- c.Assert(err, check.IsNil)
-
- s.authEmail = "active-user@arvados.local"
- s.authEmailVerified = true
- s.authName = "Fake User Name"
- s.fakeIssuer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- req.ParseForm()
- c.Logf("fakeIssuer: got req: %s %s %s", req.Method, req.URL, req.Form)
- w.Header().Set("Content-Type", "application/json")
- switch req.URL.Path {
- case "/.well-known/openid-configuration":
- json.NewEncoder(w).Encode(map[string]interface{}{
- "issuer": s.fakeIssuer.URL,
- "authorization_endpoint": s.fakeIssuer.URL + "/auth",
- "token_endpoint": s.fakeIssuer.URL + "/token",
- "jwks_uri": s.fakeIssuer.URL + "/jwks",
- "userinfo_endpoint": s.fakeIssuer.URL + "/userinfo",
- })
- case "/token":
- var clientID, clientSecret string
- auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
- authsplit := strings.Split(string(auth), ":")
- if len(authsplit) == 2 {
- clientID, _ = url.QueryUnescape(authsplit[0])
- clientSecret, _ = url.QueryUnescape(authsplit[1])
- }
- if clientID != s.validClientID || clientSecret != s.validClientSecret {
- c.Logf("fakeIssuer: expected (%q, %q) got (%q, %q)", s.validClientID, s.validClientSecret, clientID, clientSecret)
- w.WriteHeader(http.StatusUnauthorized)
- return
- }
-
- if req.Form.Get("code") != s.validCode || s.validCode == "" {
- w.WriteHeader(http.StatusUnauthorized)
- return
- }
- idToken, _ := json.Marshal(map[string]interface{}{
- "iss": s.fakeIssuer.URL,
- "aud": []string{clientID},
- "sub": "fake-user-id",
- "exp": time.Now().UTC().Add(time.Minute).Unix(),
- "iat": time.Now().UTC().Unix(),
- "nonce": "fake-nonce",
- "email": s.authEmail,
- "email_verified": s.authEmailVerified,
- "name": s.authName,
- "alt_verified": true, // for custom claim tests
- "alt_email": "alt_email@example.com", // for custom claim tests
- "alt_username": "desired-username", // for custom claim tests
- })
- json.NewEncoder(w).Encode(struct {
- AccessToken string `json:"access_token"`
- TokenType string `json:"token_type"`
- RefreshToken string `json:"refresh_token"`
- ExpiresIn int32 `json:"expires_in"`
- IDToken string `json:"id_token"`
- }{
- AccessToken: s.fakeToken(c, []byte("fake access token")),
- TokenType: "Bearer",
- RefreshToken: "test-refresh-token",
- ExpiresIn: 30,
- IDToken: s.fakeToken(c, idToken),
- })
- case "/jwks":
- json.NewEncoder(w).Encode(jose.JSONWebKeySet{
- Keys: []jose.JSONWebKey{
- {Key: s.issuerKey.Public(), Algorithm: string(jose.RS256), KeyID: ""},
- },
- })
- case "/auth":
- w.WriteHeader(http.StatusInternalServerError)
- case "/userinfo":
- w.WriteHeader(http.StatusInternalServerError)
- default:
- w.WriteHeader(http.StatusNotFound)
- }
- }))
- s.validCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
-
- s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- req.ParseForm()
- c.Logf("fakePeopleAPI: got req: %s %s %s", req.Method, req.URL, req.Form)
- w.Header().Set("Content-Type", "application/json")
- switch req.URL.Path {
- case "/v1/people/me":
- if f := req.Form.Get("personFields"); f != "emailAddresses,names" {
- w.WriteHeader(http.StatusBadRequest)
- break
- }
- json.NewEncoder(w).Encode(s.fakePeopleAPIResponse)
- default:
- w.WriteHeader(http.StatusNotFound)
- }
- }))
- s.fakePeopleAPIResponse = map[string]interface{}{}
+ s.fakeProvider = arvadostest.NewOIDCProvider(c)
+ s.fakeProvider.AuthEmail = "active-user@arvados.local"
+ s.fakeProvider.AuthEmailVerified = true
+ s.fakeProvider.AuthName = "Fake User Name"
+ s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{}
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, check.IsNil)
s.cluster.Login.Google.ClientID = "test%client$id"
s.cluster.Login.Google.ClientSecret = "test#client/secret"
s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
- s.validClientID = "test%client$id"
- s.validClientSecret = "test#client/secret"
+ s.fakeProvider.ValidClientID = "test%client$id"
+ s.fakeProvider.ValidClientSecret = "test#client/secret"
s.localdb = NewConn(s.cluster)
c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
- s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeIssuer.URL
- s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeProvider.Issuer.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
c.Check(err, check.IsNil)
target, err := url.Parse(resp.RedirectLocation)
c.Check(err, check.IsNil)
- issuerURL, _ := url.Parse(s.fakeIssuer.URL)
+ issuerURL, _ := url.Parse(s.fakeProvider.Issuer.URL)
c.Check(target.Host, check.Equals, issuerURL.Host)
q := target.Query()
c.Check(q.Get("client_id"), check.Equals, "test%client$id")
func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: "bogus-state",
})
c.Check(err, check.IsNil)
}
func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) {
- s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ s.fakeProvider.PeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprintln(w, `Error 403: accessNotConfigured`)
}))
- s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
}
func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false
- s.authEmail = "joe.smith@primary.example.com"
+ s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
s.setupPeopleAPIError(c)
state := s.startLogin(c)
_, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
c.Check(err, check.IsNil)
s.setupPeopleAPIError(c)
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
c.Check(err, check.IsNil)
func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
s.cluster.Login.Google.Enable = false
s.cluster.Login.OpenIDConnect.Enable = true
- json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
+ json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeProvider.Issuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
- s.validClientID = "oidc#client#id"
- s.validClientSecret = "oidc#client#secret"
+ s.fakeProvider.ValidClientID = "oidc#client#id"
+ s.fakeProvider.ValidClientSecret = "oidc#client#secret"
for _, trial := range []struct {
expectEmail string // "" if failure expected
setup func()
expectEmail: "user@oidc.example.com",
setup: func() {
c.Log("=== succeed because email_verified is false but not required")
- s.authEmail = "user@oidc.example.com"
- s.authEmailVerified = false
+ s.fakeProvider.AuthEmail = "user@oidc.example.com"
+ s.fakeProvider.AuthEmailVerified = false
s.cluster.Login.OpenIDConnect.EmailClaim = "email"
s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = ""
s.cluster.Login.OpenIDConnect.UsernameClaim = ""
expectEmail: "",
setup: func() {
c.Log("=== fail because email_verified is false and required")
- s.authEmail = "user@oidc.example.com"
- s.authEmailVerified = false
+ s.fakeProvider.AuthEmail = "user@oidc.example.com"
+ s.fakeProvider.AuthEmailVerified = false
s.cluster.Login.OpenIDConnect.EmailClaim = "email"
s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
s.cluster.Login.OpenIDConnect.UsernameClaim = ""
expectEmail: "user@oidc.example.com",
setup: func() {
c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim")
- s.authEmail = "user@oidc.example.com"
- s.authEmailVerified = false
+ s.fakeProvider.AuthEmail = "user@oidc.example.com"
+ s.fakeProvider.AuthEmailVerified = false
s.cluster.Login.OpenIDConnect.EmailClaim = "email"
s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
s.cluster.Login.OpenIDConnect.UsernameClaim = ""
expectEmail: "alt_email@example.com",
setup: func() {
c.Log("=== succeed with custom 'email' and 'email_verified' claims")
- s.authEmail = "bad@wrong.example.com"
- s.authEmailVerified = false
+ s.fakeProvider.AuthEmail = "bad@wrong.example.com"
+ s.fakeProvider.AuthEmailVerified = false
s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email"
s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username"
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
c.Assert(err, check.IsNil)
func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
c.Check(err, check.IsNil)
}
func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
- s.authEmail = "joe.smith@primary.example.com"
- s.fakePeopleAPIResponse = map[string]interface{}{
+ s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
"names": []map[string]interface{}{
{
"metadata": map[string]interface{}{"primary": false},
}
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
}
func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
- s.authName = "Joe P. Smith"
- s.authEmail = "joe.smith@primary.example.com"
+ s.fakeProvider.AuthName = "Joe P. Smith"
+ s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
// People API returns some additional email addresses.
func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
- s.authEmail = "joe.smith@primary.example.com"
- s.fakePeopleAPIResponse = map[string]interface{}{
+ s.fakeProvider.AuthEmail = "joe.smith@primary.example.com"
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
{
"metadata": map[string]interface{}{"verified": true},
}
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
// Primary address is not the one initially returned by oidc.
func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
- s.authEmail = "joe.smith@alternate.example.com"
- s.fakePeopleAPIResponse = map[string]interface{}{
+ s.fakeProvider.AuthEmail = "joe.smith@alternate.example.com"
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
{
"metadata": map[string]interface{}{"verified": true, "primary": true},
}
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
authinfo := getCallbackAuthInfo(c, s.railsSpy)
}
func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
- s.authEmail = "joe.smith@unverified.example.com"
- s.authEmailVerified = false
- s.fakePeopleAPIResponse = map[string]interface{}{
+ s.fakeProvider.AuthEmail = "joe.smith@unverified.example.com"
+ s.fakeProvider.AuthEmailVerified = false
+ s.fakeProvider.PeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
{
"metadata": map[string]interface{}{"verified": true},
}
state := s.startLogin(c)
s.localdb.Login(context.Background(), arvados.LoginOptions{
- Code: s.validCode,
+ Code: s.fakeProvider.ValidCode,
State: state,
})
return
}
-func (s *OIDCLoginSuite) fakeToken(c *check.C, payload []byte) string {
- signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: s.issuerKey}, nil)
- if err != nil {
- c.Error(err)
- }
- object, err := signer.Sign(payload)
- if err != nil {
- c.Error(err)
- }
- t, err := object.CompactSerialize()
- if err != nil {
- c.Error(err)
- }
- c.Logf("fakeToken(%q) == %q", payload, t)
- return t
-}
-
func getCallbackAuthInfo(c *check.C, railsSpy *arvadostest.Proxy) (authinfo rpc.UserSessionAuthInfo) {
for _, dump := range railsSpy.RequestDumps {
c.Logf("spied request: %q", dump)
redir += '?'
}
const respj = await resp.json()
- document.location = redir + "api_token=" + respj.api_token
+ document.location = redir + "api_token=v2/" + respj.uuid + "/" + respj.api_token
}
</script>
</head>
"git.arvados.org/arvados.git/sdk/go/arvados"
)
-// For now, FindRailsAPI always uses the rails API running on this
-// node.
+// FindRailsAPI always uses the rails API running on this node, for now.
func FindRailsAPI(cluster *arvados.Cluster) (*url.URL, bool, error) {
var best *url.URL
for target := range cluster.Services.RailsAPI.InternalURLs {
var _ = check.Suite(&RPCSuite{})
-const contextKeyTestTokens = "testTokens"
+type key int
+
+const (
+ contextKeyTestTokens key = iota
+)
type RPCSuite struct {
log logrus.FieldLogger
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package costanalyzer
+
+import (
+ "io"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+type NoPrefixFormatter struct{}
+
+func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ return []byte(entry.Message), nil
+}
+
+// RunCommand implements the subcommand "costanalyzer <collection> <collection> ..."
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.Error("\n" + err.Error() + "\n")
+ }
+ }()
+
+ logger.SetFormatter(new(NoPrefixFormatter))
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ exitcode, err := costanalyzer(prog, args, loader, logger, stdout, stderr)
+
+ return exitcode
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/sirupsen/logrus"
+)
+
+type nodeInfo struct {
+ // Legacy (records created by Arvados Node Manager with Arvados <= 1.4.3)
+ Properties struct {
+ CloudNode struct {
+ Price float64
+ Size string
+ } `json:"cloud_node"`
+ }
+ // Modern
+ ProviderType string
+ Price float64
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+ return ""
+}
+
+func (i *arrayFlags) Set(value string) error {
+ for _, s := range strings.Split(value, ",") {
+ *i = append(*i, s)
+ }
+ return nil
+}
+
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitCode int, uuids arrayFlags, resultsDir string, cache bool, err error) {
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `
+Usage:
+ %s [options ...] <uuid> ...
+
+ This program analyzes the cost of Arvados container requests. For each uuid
+ supplied, it creates a CSV report that lists all the containers used to
+ fulfill the container request, together with the machine type and cost of
+ each container. At least one uuid must be specified.
+
+ When supplied with the uuid of a container request, it will calculate the
+ cost of that container request and all its children.
+
+ When supplied with the uuid of a collection, it will see if there is a
+ container_request uuid in the properties of the collection, and if so, it
+ will calculate the cost of that container request and all its children.
+
+ When supplied with a project uuid or when supplied with multiple container
+ request or collection uuids, it will create a CSV report for each supplied
+ uuid, as well as a CSV file with aggregate cost accounting for all supplied
+ uuids. The aggregate cost report takes container reuse into account: if a
+ container was reused between several container requests, its cost will only
+ be counted once.
+
+ To get the node costs, the progam queries the Arvados API for current cost
+ data for each node type used. This means that the reported cost always
+ reflects the cost data as currently defined in the Arvados API configuration
+ file.
+
+ Caveats:
+ - the Arvados API configuration cost data may be out of sync with the cloud
+ provider.
+ - when generating reports for older container requests, the cost data in the
+ Arvados API configuration file may have changed since the container request
+ was fulfilled. This program uses the cost data stored at the time of the
+ execution of the container, stored in the 'node.json' file in its log
+ collection.
+
+ In order to get the data for the uuids supplied, the ARVADOS_API_HOST and
+ ARVADOS_API_TOKEN environment variables must be set.
+
+ This program prints the total dollar amount from the aggregate cost
+ accounting across all provided uuids on stdout.
+
+ When the '-output' option is specified, a set of CSV files with cost details
+ will be written to the provided directory.
+
+Options:
+`, prog)
+ flags.PrintDefaults()
+ }
+ loglevel := flags.String("log-level", "info", "logging `level` (debug, info, ...)")
+ flags.StringVar(&resultsDir, "output", "", "output `directory` for the CSV reports")
+ flags.BoolVar(&cache, "cache", true, "create and use a local disk cache of Arvados objects")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ exitCode = 1
+ return
+ } else if err != nil {
+ exitCode = 2
+ return
+ }
+ uuids = flags.Args()
+
+ if len(uuids) < 1 {
+ flags.Usage()
+ err = fmt.Errorf("error: no uuid(s) provided")
+ exitCode = 2
+ return
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ exitCode = 2
+ return
+ }
+ logger.SetLevel(lvl)
+ if !cache {
+ logger.Debug("Caching disabled\n")
+ }
+ return
+}
+
+func ensureDirectory(logger *logrus.Logger, dir string) (err error) {
+ statData, err := os.Stat(dir)
+ if os.IsNotExist(err) {
+ err = os.MkdirAll(dir, 0700)
+ if err != nil {
+ return fmt.Errorf("error creating directory %s: %s", dir, err.Error())
+ }
+ } else {
+ if !statData.IsDir() {
+ return fmt.Errorf("the path %s is not a directory", dir)
+ }
+ }
+ return
+}
+
+func addContainerLine(logger *logrus.Logger, node nodeInfo, cr arvados.ContainerRequest, container arvados.Container) (csv string, cost float64) {
+ csv = cr.UUID + ","
+ csv += cr.Name + ","
+ csv += container.UUID + ","
+ csv += string(container.State) + ","
+ if container.StartedAt != nil {
+ csv += container.StartedAt.String() + ","
+ } else {
+ csv += ","
+ }
+
+ var delta time.Duration
+ if container.FinishedAt != nil {
+ csv += container.FinishedAt.String() + ","
+ delta = container.FinishedAt.Sub(*container.StartedAt)
+ csv += strconv.FormatFloat(delta.Seconds(), 'f', 0, 64) + ","
+ } else {
+ csv += ",,"
+ }
+ var price float64
+ var size string
+ if node.Properties.CloudNode.Price != 0 {
+ price = node.Properties.CloudNode.Price
+ size = node.Properties.CloudNode.Size
+ } else {
+ price = node.Price
+ size = node.ProviderType
+ }
+ cost = delta.Seconds() / 3600 * price
+ csv += size + "," + strconv.FormatFloat(price, 'f', 8, 64) + "," + strconv.FormatFloat(cost, 'f', 8, 64) + "\n"
+ return
+}
+
+func loadCachedObject(logger *logrus.Logger, file string, uuid string, object interface{}) (reload bool) {
+ reload = true
+ if strings.Contains(uuid, "-j7d0g-") || strings.Contains(uuid, "-4zz18-") {
+ // We do not cache projects or collections, they have no final state
+ return
+ }
+ // See if we have a cached copy of this object
+ _, err := os.Stat(file)
+ if err != nil {
+ return
+ }
+ data, err := ioutil.ReadFile(file)
+ if err != nil {
+ logger.Errorf("error reading %q: %s", file, err)
+ return
+ }
+ err = json.Unmarshal(data, &object)
+ if err != nil {
+ logger.Errorf("failed to unmarshal json: %s: %s", data, err)
+ return
+ }
+
+ // See if it is in a final state, if that makes sense
+ switch v := object.(type) {
+ case *arvados.ContainerRequest:
+ if v.State == arvados.ContainerRequestStateFinal {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ }
+ case *arvados.Container:
+ if v.State == arvados.ContainerStateComplete || v.State == arvados.ContainerStateCancelled {
+ reload = false
+ logger.Debugf("Loaded object %s from local cache (%s)\n", uuid, file)
+ }
+ }
+ return
+}
+
+// Load an Arvados object.
+func loadObject(logger *logrus.Logger, ac *arvados.Client, path string, uuid string, cache bool, object interface{}) (err error) {
+ file := uuid + ".json"
+
+ var reload bool
+ var cacheDir string
+
+ if !cache {
+ reload = true
+ } else {
+ homeDir, err := os.UserHomeDir()
+ if err != nil {
+ reload = true
+ logger.Info("Unable to determine current user home directory, not using cache")
+ } else {
+ cacheDir = homeDir + "/.cache/arvados/costanalyzer/"
+ err = ensureDirectory(logger, cacheDir)
+ if err != nil {
+ reload = true
+ logger.Infof("Unable to create cache directory at %s, not using cache: %s", cacheDir, err.Error())
+ } else {
+ reload = loadCachedObject(logger, cacheDir+file, uuid, object)
+ }
+ }
+ }
+ if !reload {
+ return
+ }
+
+ if strings.Contains(uuid, "-j7d0g-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/groups/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-xvhdp-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/container_requests/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-dz642-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/containers/"+uuid, nil, nil)
+ } else if strings.Contains(uuid, "-4zz18-") {
+ err = ac.RequestAndDecode(&object, "GET", "arvados/v1/collections/"+uuid, nil, nil)
+ } else {
+ err = fmt.Errorf("unsupported object type with UUID %q:\n %s", uuid, err)
+ return
+ }
+ if err != nil {
+ err = fmt.Errorf("error loading object with UUID %q:\n %s", uuid, err)
+ return
+ }
+ encoded, err := json.MarshalIndent(object, "", " ")
+ if err != nil {
+ err = fmt.Errorf("error marshaling object with UUID %q:\n %s", uuid, err)
+ return
+ }
+ if cacheDir != "" {
+ err = ioutil.WriteFile(cacheDir+file, encoded, 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file %s:\n %s", file, err)
+ return
+ }
+ }
+ return
+}
+
+func getNode(arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, cr arvados.ContainerRequest) (node nodeInfo, err error) {
+ if cr.LogUUID == "" {
+ err = errors.New("no log collection")
+ return
+ }
+
+ var collection arvados.Collection
+ err = ac.RequestAndDecode(&collection, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ if err != nil {
+ err = fmt.Errorf("error getting collection: %s", err)
+ return
+ }
+
+ var fs arvados.CollectionFileSystem
+ fs, err = collection.FileSystem(ac, kc)
+ if err != nil {
+ err = fmt.Errorf("error opening collection as filesystem: %s", err)
+ return
+ }
+ var f http.File
+ f, err = fs.Open("node.json")
+ if err != nil {
+ err = fmt.Errorf("error opening file 'node.json' in collection %s: %s", cr.LogUUID, err)
+ return
+ }
+
+ err = json.NewDecoder(f).Decode(&node)
+ if err != nil {
+ err = fmt.Errorf("error reading file 'node.json' in collection %s: %s", cr.LogUUID, err)
+ return
+ }
+ return
+}
+
+func handleProject(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+ cost = make(map[string]float64)
+
+ var project arvados.Group
+ err = loadObject(logger, ac, uuid, uuid, cache, &project)
+ if err != nil {
+ return nil, fmt.Errorf("error loading object %s: %s", uuid, err.Error())
+ }
+
+ var childCrs map[string]interface{}
+ filterset := []arvados.Filter{
+ {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: project.UUID,
+ },
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: nil,
+ },
+ }
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
+ }
+ if value, ok := childCrs["items"]; ok {
+ logger.Infof("Collecting top level container requests in project %s\n", uuid)
+ items := value.([]interface{})
+ for _, item := range items {
+ itemMap := item.(map[string]interface{})
+ crCsv, err := generateCrCsv(logger, itemMap["uuid"].(string), arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ return nil, fmt.Errorf("error generating container_request CSV: %s", err.Error())
+ }
+ for k, v := range crCsv {
+ cost[k] = v
+ }
+ }
+ } else {
+ logger.Infof("No top level container requests found in project %s\n", uuid)
+ }
+ return
+}
+
+func generateCrCsv(logger *logrus.Logger, uuid string, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, resultsDir string, cache bool) (cost map[string]float64, err error) {
+
+ cost = make(map[string]float64)
+
+ csv := "CR UUID,CR name,Container UUID,State,Started At,Finished At,Duration in seconds,Compute node type,Hourly node cost,Total cost\n"
+ var tmpCsv string
+ var tmpTotalCost float64
+ var totalCost float64
+
+ var crUUID = uuid
+ if strings.Contains(uuid, "-4zz18-") {
+ // This is a collection, find the associated container request (if any)
+ var c arvados.Collection
+ err = loadObject(logger, ac, uuid, uuid, cache, &c)
+ if err != nil {
+ return nil, fmt.Errorf("error loading collection object %s: %s", uuid, err)
+ }
+ value, ok := c.Properties["container_request"]
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property", uuid)
+ }
+ crUUID, ok = value.(string)
+ if !ok {
+ return nil, fmt.Errorf("error: collection %s does not have a 'container_request' property of the string type", uuid)
+ }
+ }
+
+ // This is a container request, find the container
+ var cr arvados.ContainerRequest
+ err = loadObject(logger, ac, crUUID, crUUID, cache, &cr)
+ if err != nil {
+ return nil, fmt.Errorf("error loading cr object %s: %s", uuid, err)
+ }
+ var container arvados.Container
+ err = loadObject(logger, ac, crUUID, cr.ContainerUUID, cache, &container)
+ if err != nil {
+ return nil, fmt.Errorf("error loading container object %s: %s", cr.ContainerUUID, err)
+ }
+
+ topNode, err := getNode(arv, ac, kc, cr)
+ if err != nil {
+ return nil, fmt.Errorf("error getting node %s: %s", cr.UUID, err)
+ }
+ tmpCsv, totalCost = addContainerLine(logger, topNode, cr, container)
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+ cost[container.UUID] = totalCost
+
+ // Find all container requests that have the container we found above as requesting_container_uuid
+ var childCrs arvados.ContainerRequestList
+ filterset := []arvados.Filter{
+ {
+ Attr: "requesting_container_uuid",
+ Operator: "=",
+ Operand: container.UUID,
+ }}
+ err = ac.RequestAndDecode(&childCrs, "GET", "arvados/v1/container_requests", nil, map[string]interface{}{
+ "filters": filterset,
+ "limit": 10000,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("error querying container_requests: %s", err.Error())
+ }
+ logger.Infof("Collecting child containers for container request %s", crUUID)
+ for _, cr2 := range childCrs.Items {
+ logger.Info(".")
+ node, err := getNode(arv, ac, kc, cr2)
+ if err != nil {
+ return nil, fmt.Errorf("error getting node %s: %s", cr2.UUID, err)
+ }
+ logger.Debug("\nChild container: " + cr2.ContainerUUID + "\n")
+ var c2 arvados.Container
+ err = loadObject(logger, ac, cr.UUID, cr2.ContainerUUID, cache, &c2)
+ if err != nil {
+ return nil, fmt.Errorf("error loading object %s: %s", cr2.ContainerUUID, err)
+ }
+ tmpCsv, tmpTotalCost = addContainerLine(logger, node, cr2, c2)
+ cost[cr2.ContainerUUID] = tmpTotalCost
+ csv += tmpCsv
+ totalCost += tmpTotalCost
+ }
+ logger.Info(" done\n")
+
+ csv += "TOTAL,,,,,,,,," + strconv.FormatFloat(totalCost, 'f', 8, 64) + "\n"
+
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ fName := resultsDir + "/" + crUUID + ".csv"
+ err = ioutil.WriteFile(fName, []byte(csv), 0644)
+ if err != nil {
+ return nil, fmt.Errorf("error writing file with path %s: %s", fName, err.Error())
+ }
+ logger.Infof("\nUUID report in %s\n\n", fName)
+ }
+
+ return
+}
+
+func costanalyzer(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int, err error) {
+ exitcode, uuids, resultsDir, cache, err := parseFlags(prog, args, loader, logger, stderr)
+ if exitcode != 0 {
+ return
+ }
+ if resultsDir != "" {
+ err = ensureDirectory(logger, resultsDir)
+ if err != nil {
+ exitcode = 3
+ return
+ }
+ }
+
+ // Arvados Client setup
+ arv, err := arvadosclient.MakeArvadosClient()
+ if err != nil {
+ err = fmt.Errorf("error creating Arvados object: %s", err)
+ exitcode = 1
+ return
+ }
+ kc, err := keepclient.MakeKeepClient(arv)
+ if err != nil {
+ err = fmt.Errorf("error creating Keep object: %s", err)
+ exitcode = 1
+ return
+ }
+
+ ac := arvados.NewClientFromEnv()
+
+ cost := make(map[string]float64)
+ for _, uuid := range uuids {
+ if strings.Contains(uuid, "-j7d0g-") {
+ // This is a project (group)
+ cost, err = handleProject(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ exitcode = 1
+ return
+ }
+ for k, v := range cost {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-xvhdp-") || strings.Contains(uuid, "-4zz18-") {
+ // This is a container request
+ var crCsv map[string]float64
+ crCsv, err = generateCrCsv(logger, uuid, arv, ac, kc, resultsDir, cache)
+ if err != nil {
+ err = fmt.Errorf("error generating CSV for uuid %s: %s", uuid, err.Error())
+ exitcode = 2
+ return
+ }
+ for k, v := range crCsv {
+ cost[k] = v
+ }
+ } else if strings.Contains(uuid, "-tpzed-") {
+ // This is a user. The "Home" project for a user is not a real project.
+ // It is identified by the user uuid. As such, cost analysis for the
+ // "Home" project is not supported by this program. Skip this uuid, but
+ // keep going.
+ logger.Errorf("cost analysis is not supported for the 'Home' project: %s", uuid)
+ } else {
+ logger.Errorf("this argument does not look like a uuid: %s\n", uuid)
+ exitcode = 3
+ return
+ }
+ }
+
+ if len(cost) == 0 {
+ logger.Info("Nothing to do!\n")
+ return
+ }
+
+ var csv string
+
+ csv = "# Aggregate cost accounting for uuids:\n"
+ for _, uuid := range uuids {
+ csv += "# " + uuid + "\n"
+ }
+
+ var total float64
+ for k, v := range cost {
+ csv += k + "," + strconv.FormatFloat(v, 'f', 8, 64) + "\n"
+ total += v
+ }
+
+ csv += "TOTAL," + strconv.FormatFloat(total, 'f', 8, 64) + "\n"
+
+ if resultsDir != "" {
+ // Write the resulting CSV file
+ aFile := resultsDir + "/" + time.Now().Format("2006-01-02-15-04-05") + "-aggregate-costaccounting.csv"
+ err = ioutil.WriteFile(aFile, []byte(csv), 0644)
+ if err != nil {
+ err = fmt.Errorf("error writing file with path %s: %s", aFile, err.Error())
+ exitcode = 1
+ return
+ }
+ logger.Infof("Aggregate cost accounting for all supplied uuids in %s\n", aFile)
+ }
+
+ // Output the total dollar amount on stdout
+ fmt.Fprintf(stdout, "%s\n", strconv.FormatFloat(total, 'f', 8, 64))
+
+ return
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package costanalyzer
+
+import (
+ "bytes"
+ "io"
+ "io/ioutil"
+ "os"
+ "regexp"
+ "testing"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the database so they don't affect subsequent tests.
+ arvadostest.ResetEnv()
+}
+
+func (s *Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+
+ // Get the various arvados, arvadosclient, and keep client objects
+ ac := arvados.NewClientFromEnv()
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, check.Equals, nil)
+ arv.ApiToken = arvadostest.ActiveToken
+ kc, err := keepclient.MakeKeepClient(arv)
+ c.Assert(err, check.Equals, nil)
+
+ standardE4sV3JSON := `{
+ "Name": "Standard_E4s_v3",
+ "ProviderType": "Standard_E4s_v3",
+ "VCPUs": 4,
+ "RAM": 34359738368,
+ "Scratch": 64000000000,
+ "IncludedScratch": 64000000000,
+ "AddedScratch": 0,
+ "Price": 0.292,
+ "Preemptible": false
+}`
+ standardD32sV3JSON := `{
+ "Name": "Standard_D32s_v3",
+ "ProviderType": "Standard_D32s_v3",
+ "VCPUs": 32,
+ "RAM": 137438953472,
+ "Scratch": 256000000000,
+ "IncludedScratch": 256000000000,
+ "AddedScratch": 0,
+ "Price": 1.76,
+ "Preemptible": false
+}`
+
+ standardA1V2JSON := `{
+ "Name": "a1v2",
+ "ProviderType": "Standard_A1_v2",
+ "VCPUs": 1,
+ "RAM": 2147483648,
+ "Scratch": 10000000000,
+ "IncludedScratch": 10000000000,
+ "AddedScratch": 0,
+ "Price": 0.043,
+ "Preemptible": false
+}`
+
+ standardA2V2JSON := `{
+ "Name": "a2v2",
+ "ProviderType": "Standard_A2_v2",
+ "VCPUs": 2,
+ "RAM": 4294967296,
+ "Scratch": 20000000000,
+ "IncludedScratch": 20000000000,
+ "AddedScratch": 0,
+ "Price": 0.091,
+ "Preemptible": false
+}`
+
+ legacyD1V2JSON := `{
+ "properties": {
+ "cloud_node": {
+ "price": 0.073001,
+ "size": "Standard_D1_v2"
+ },
+ "total_cpu_cores": 1,
+ "total_ram_mb": 3418,
+ "total_scratch_mb": 51170
+ }
+}`
+
+ // Our fixtures do not actually contain file contents. Populate the log collections we're going to use with the node.json file
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID, arvadostest.LogCollectionUUID, standardE4sV3JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedContainerRequestUUID2, arvadostest.LogCollectionUUID2, standardD32sV3JSON)
+
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.DiagnosticsContainerRequest1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsContainerRequest2UUID, arvadostest.DiagnosticsContainerRequest2LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher1ContainerRequestUUID, arvadostest.Hasher1LogCollectionUUID, standardA1V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher2ContainerRequestUUID, arvadostest.Hasher2LogCollectionUUID, standardA2V2JSON)
+ createNodeJSON(c, arv, ac, kc, arvadostest.CompletedDiagnosticsHasher3ContainerRequestUUID, arvadostest.Hasher3LogCollectionUUID, legacyD1V2JSON)
+}
+
+func createNodeJSON(c *check.C, arv *arvadosclient.ArvadosClient, ac *arvados.Client, kc *keepclient.KeepClient, crUUID string, logUUID string, nodeJSON string) {
+ // Get the CR
+ var cr arvados.ContainerRequest
+ err := ac.RequestAndDecode(&cr, "GET", "arvados/v1/container_requests/"+crUUID, nil, nil)
+ c.Assert(err, check.Equals, nil)
+ c.Assert(cr.LogUUID, check.Equals, logUUID)
+
+ // Get the log collection
+ var coll arvados.Collection
+ err = ac.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+cr.LogUUID, nil, nil)
+ c.Assert(err, check.IsNil)
+
+ // Create a node.json file -- the fixture doesn't actually contain the contents of the collection.
+ fs, err := coll.FileSystem(ac, kc)
+ c.Assert(err, check.IsNil)
+ f, err := fs.OpenFile("node.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, nodeJSON)
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+
+ // Flush the data to Keep
+ mtxt, err := fs.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ c.Assert(mtxt, check.NotNil)
+
+ // Update collection record
+ err = ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+cr.LogUUID, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": mtxt,
+ },
+ })
+ c.Assert(err, check.IsNil)
+}
+
+func (*Suite) TestUsage(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-help", "-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with 1 container request uuid
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestCollectionUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+
+ resultsDir := c.MkDir()
+ // Run costanalyzer with 1 collection uuid, without 'container_request' property
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 2)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*does not have a 'container_request' property.*")
+
+ // Update the collection, attach a 'container_request' property
+ ac := arvados.NewClientFromEnv()
+ var coll arvados.Collection
+
+ // Update collection record
+ err := ac.RequestAndDecode(&coll, "PUT", "arvados/v1/collections/"+arvadostest.FooCollection, nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "properties": map[string]interface{}{
+ "container_request": arvadostest.CompletedContainerRequestUUID,
+ },
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Run costanalyzer with 1 collection uuid
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,7.01302889")
+}
+
+func (*Suite) TestDoubleContainerRequestUUID(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ resultsDir := c.MkDir()
+ // Run costanalyzer with 2 container request uuids
+ exitcode := Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedContainerRequestUUID, arvadostest.CompletedContainerRequestUUID2}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Now move both container requests into an existing project, and then re-run
+ // the analysis with the project uuid. The results should be identical.
+ ac := arvados.NewClientFromEnv()
+ var cr arvados.ContainerRequest
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ err = ac.RequestAndDecode(&cr, "PUT", "arvados/v1/container_requests/"+arvadostest.CompletedContainerRequestUUID2, nil, map[string]interface{}{
+ "container_request": map[string]interface{}{
+ "owner_uuid": arvadostest.AProjectUUID,
+ },
+ })
+ c.Assert(err, check.IsNil)
+
+ // Run costanalyzer with the project uuid
+ resultsDir = c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-cache=false", "-log-level", "debug", "-output", resultsDir, arvadostest.AProjectUUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,7.01302889")
+
+ uuidReport2, err = ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedContainerRequestUUID2 + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,42.27031111")
+
+ re = regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches = re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err = ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,49.28334000")
+}
+
+func (*Suite) TestMultipleContainerRequestUUIDWithReuse(c *check.C) {
+ var stdout, stderr bytes.Buffer
+ // Run costanalyzer with 2 container request uuids, without output directory specified
+ exitcode := Command.RunCommand("costanalyzer.test", []string{arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Not(check.Matches), "(?ms).*supplied uuids in .*")
+
+ // Check that the total amount was printed to stdout
+ c.Check(stdout.String(), check.Matches, "0.01492030\n")
+
+ stdout.Truncate(0)
+ stderr.Truncate(0)
+
+ // Run costanalyzer with 2 container request uuids
+ resultsDir := c.MkDir()
+ exitcode = Command.RunCommand("costanalyzer.test", []string{"-output", resultsDir, arvadostest.CompletedDiagnosticsContainerRequest1UUID, arvadostest.CompletedDiagnosticsContainerRequest2UUID}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Assert(stderr.String(), check.Matches, "(?ms).*supplied uuids in .*")
+
+ uuidReport, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest1UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00916192")
+
+ uuidReport2, err := ioutil.ReadFile(resultsDir + "/" + arvadostest.CompletedDiagnosticsContainerRequest2UUID + ".csv")
+ c.Assert(err, check.IsNil)
+ c.Check(string(uuidReport2), check.Matches, "(?ms).*TOTAL,,,,,,,,,0.00588088")
+
+ re := regexp.MustCompile(`(?ms).*supplied uuids in (.*?)\n`)
+ matches := re.FindStringSubmatch(stderr.String()) // matches[1] contains a string like 'results/2020-11-02-18-57-45-aggregate-costaccounting.csv'
+
+ aggregateCostReport, err := ioutil.ReadFile(matches[1])
+ c.Assert(err, check.IsNil)
+
+ c.Check(string(aggregateCostReport), check.Matches, "(?ms).*TOTAL,0.01492030")
+}
var pi procinfo
err = json.NewDecoder(f).Decode(&pi)
if err != nil {
- return fmt.Errorf("decode %s: %s\n", path, err)
+ return fmt.Errorf("decode %s: %s", path, err)
}
if pi.UUID != uuid || pi.PID == 0 {
return nil
}
-// List UUIDs of active crunch-run processes.
+// ListProcesses lists UUIDs of active crunch-run processes.
func ListProcesses(stdout, stderr io.Writer) int {
// filepath.Walk does not follow symlinks, so we must walk
// lockdir+"/." in case lockdir itself is a symlink.
}
for bind := range runner.SecretMounts {
if _, ok := runner.Container.Mounts[bind]; ok {
- return fmt.Errorf("Secret mount %q conflicts with regular mount", bind)
+ return fmt.Errorf("secret mount %q conflicts with regular mount", bind)
}
if runner.SecretMounts[bind].Kind != "json" &&
runner.SecretMounts[bind].Kind != "text" {
- return fmt.Errorf("Secret mount %q type is %q but only 'json' and 'text' are permitted.",
+ return fmt.Errorf("secret mount %q type is %q but only 'json' and 'text' are permitted",
bind, runner.SecretMounts[bind].Kind)
}
binds = append(binds, bind)
if bind == "stdout" || bind == "stderr" {
// Is it a "file" mount kind?
if mnt.Kind != "file" {
- return fmt.Errorf("Unsupported mount kind '%s' for %s. Only 'file' is supported.", mnt.Kind, bind)
+ return fmt.Errorf("unsupported mount kind '%s' for %s: only 'file' is supported", mnt.Kind, bind)
}
// Does path start with OutputPath?
if bind == "stdin" {
// Is it a "collection" mount kind?
if mnt.Kind != "collection" && mnt.Kind != "json" {
- return fmt.Errorf("Unsupported mount kind '%s' for stdin. Only 'collection' or 'json' are supported.", mnt.Kind)
+ return fmt.Errorf("unsupported mount kind '%s' for stdin: only 'collection' and 'json' are supported", mnt.Kind)
}
}
if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && bind != runner.Container.OutputPath+"/" {
if mnt.Kind != "collection" && mnt.Kind != "text" && mnt.Kind != "json" {
- return fmt.Errorf("Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
+ return fmt.Errorf("only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path for %q, was %q", bind, mnt.Kind)
}
}
case mnt.Kind == "collection" && bind != "stdin":
var src string
if mnt.UUID != "" && mnt.PortableDataHash != "" {
- return fmt.Errorf("Cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
+ return fmt.Errorf("cannot specify both 'uuid' and 'portable_data_hash' for a collection mount")
}
if mnt.UUID != "" {
if mnt.Writable {
- return fmt.Errorf("Writing to existing collections currently not permitted.")
+ return fmt.Errorf("writing to existing collections currently not permitted")
}
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Can never write to a collection specified by portable data hash")
+ return fmt.Errorf("can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
if idx > 0 {
src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount)
arvMountCmd = append(arvMountCmd, "--mount-tmp")
arvMountCmd = append(arvMountCmd, fmt.Sprintf("tmp%d", tmpcount))
- tmpcount += 1
+ tmpcount++
}
if mnt.Writable {
if bind == runner.Container.OutputPath {
var tmpdir string
tmpdir, err = runner.MkTempDir(runner.parentTemp, "tmp")
if err != nil {
- return fmt.Errorf("While creating mount temp dir: %v", err)
+ return fmt.Errorf("while creating mount temp dir: %v", err)
}
st, staterr := os.Stat(tmpdir)
if staterr != nil {
- return fmt.Errorf("While Stat on temp dir: %v", staterr)
+ return fmt.Errorf("while Stat on temp dir: %v", staterr)
}
err = os.Chmod(tmpdir, st.Mode()|os.ModeSetgid|0777)
if staterr != nil {
- return fmt.Errorf("While Chmod temp dir: %v", err)
+ return fmt.Errorf("while Chmod temp dir: %v", err)
}
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", tmpdir, bind))
if bind == runner.Container.OutputPath {
}
if runner.HostOutputDir == "" {
- return fmt.Errorf("Output path does not correspond to a writable mount point")
+ return fmt.Errorf("output path does not correspond to a writable mount point")
}
if wantAPI := runner.Container.RuntimeConstraints.API; needCertMount && wantAPI != nil && *wantAPI {
runner.ArvMount, err = runner.RunArvMount(arvMountCmd, token)
if err != nil {
- return fmt.Errorf("While trying to start arv-mount: %v", err)
+ return fmt.Errorf("while trying to start arv-mount: %v", err)
}
for _, p := range collectionPaths {
_, err = os.Stat(p)
if err != nil {
- return fmt.Errorf("While checking that input files exist: %v", err)
+ return fmt.Errorf("while checking that input files exist: %v", err)
}
}
for _, cp := range copyFiles {
st, err := os.Stat(cp.src)
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
if st.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
}
return os.Chmod(target, walkinfo.Mode()|os.ModeSetgid|0777)
} else {
- return fmt.Errorf("Source %q is not a regular file or directory", cp.src)
+ return fmt.Errorf("source %q is not a regular file or directory", cp.src)
}
})
} else if st.Mode().IsRegular() {
}
}
if err != nil {
- return fmt.Errorf("While staging writable file from %q to %q: %v", cp.src, cp.bind, err)
+ return fmt.Errorf("while staging writable file from %q to %q: %v", cp.src, cp.bind, err)
}
}
// If stdin mount is provided, attach it to the docker container
var stdinRdr arvados.File
- var stdinJson []byte
+ var stdinJSON []byte
if stdinMnt, ok := runner.Container.Mounts["stdin"]; ok {
if stdinMnt.Kind == "collection" {
var stdinColl arvados.Collection
- collId := stdinMnt.UUID
- if collId == "" {
- collId = stdinMnt.PortableDataHash
+ collID := stdinMnt.UUID
+ if collID == "" {
+ collID = stdinMnt.PortableDataHash
}
- err = runner.ContainerArvClient.Get("collections", collId, nil, &stdinColl)
+ err = runner.ContainerArvClient.Get("collections", collID, nil, &stdinColl)
if err != nil {
return fmt.Errorf("While getting stdin collection: %v", err)
}
return fmt.Errorf("While getting stdin collection path %v: %v", stdinMnt.Path, err)
}
} else if stdinMnt.Kind == "json" {
- stdinJson, err = json.Marshal(stdinMnt.Content)
+ stdinJSON, err = json.Marshal(stdinMnt.Content)
if err != nil {
return fmt.Errorf("While encoding stdin json data: %v", err)
}
}
}
- stdinUsed := stdinRdr != nil || len(stdinJson) != 0
+ stdinUsed := stdinRdr != nil || len(stdinJSON) != 0
response, err := runner.Docker.ContainerAttach(context.TODO(), runner.ContainerID,
dockertypes.ContainerAttachOptions{Stream: true, Stdin: stdinUsed, Stdout: true, Stderr: true})
if err != nil {
stdinRdr.Close()
response.CloseWrite()
}()
- } else if len(stdinJson) != 0 {
+ } else if len(stdinJSON) != 0 {
go func() {
- _, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
+ _, err := io.Copy(response.Conn, bytes.NewReader(stdinJSON))
if err != nil {
runner.CrunchLog.Printf("While writing stdin json to docker container: %v", err)
runner.stop(nil)
}
}
- containerId := flags.Arg(0)
+ containerID := flags.Arg(0)
switch {
case *detach && !ignoreDetachFlag:
- return Detach(containerId, prog, args, os.Stdout, os.Stderr)
+ return Detach(containerID, prog, args, os.Stdout, os.Stderr)
case *kill >= 0:
- return KillProcess(containerId, syscall.Signal(*kill), os.Stdout, os.Stderr)
+ return KillProcess(containerID, syscall.Signal(*kill), os.Stdout, os.Stderr)
case *list:
return ListProcesses(os.Stdout, os.Stderr)
}
- if containerId == "" {
+ if containerID == "" {
log.Printf("usage: %s [options] UUID", prog)
return 1
}
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Printf("%s: %v", containerId, err)
+ log.Printf("%s: %v", containerID, err)
return 1
}
api.Retries = 8
kc, kcerr := keepclient.MakeKeepClient(api)
if kcerr != nil {
- log.Printf("%s: %v", containerId, kcerr)
+ log.Printf("%s: %v", containerID, kcerr)
return 1
}
kc.BlockCache = &keepclient.BlockCache{MaxBlocks: 2}
// minimum version we want to support.
docker, dockererr := dockerclient.NewClient(dockerclient.DefaultDockerHost, "1.21", nil, nil)
- cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerId)
+ cr, err := NewContainerRunner(arvados.NewClientFromEnv(), api, kc, docker, containerID)
if err != nil {
log.Print(err)
return 1
}
if dockererr != nil {
- cr.CrunchLog.Printf("%s: %v", containerId, dockererr)
+ cr.CrunchLog.Printf("%s: %v", containerID, dockererr)
cr.checkBrokenNode(dockererr)
cr.CrunchLog.Close()
return 1
}
- parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerId+".")
+ parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
if tmperr != nil {
- log.Printf("%s: %v", containerId, tmperr)
+ log.Printf("%s: %v", containerID, tmperr)
return 1
}
}
if runerr != nil {
- log.Printf("%s: %v", containerId, runerr)
+ log.Printf("%s: %v", containerID, runerr)
return 1
}
return 0
var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
-var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
+var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
if err != nil {
return dockertypes.ImageLoadResponse{}, err
}
- t.imageLoaded = hwImageId
+ t.imageLoaded = hwImageID
return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
}
}
func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
- if filename == hwImageId+".tar" {
+ if filename == hwImageID+".tar" {
rdr := ioutil.NopCloser(&bytes.Buffer{})
client.Called = true
return FileWrapper{rdr, 1321984}, nil
cr.ContainerArvClient = &ArvTestClient{}
cr.ContainerKeepClient = kc
- _, err = cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ _, err = cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
c.Check(err, IsNil)
- _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
+ _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID)
c.Check(err, NotNil)
cr.Container.ContainerImage = hwPDH
c.Check(err, IsNil)
defer func() {
- cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ cr.Docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
}()
c.Check(kc.Called, Equals, true)
- c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
+ c.Check(cr.ContainerConfig.Image, Equals, hwImageID)
- _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
+ _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageID)
c.Check(err, IsNil)
// (2) Test using image that's already loaded
err = cr.LoadImage()
c.Check(err, IsNil)
c.Check(kc.Called, Equals, false)
- c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
+ c.Check(cr.ContainerConfig.Image, Equals, hwImageID)
}
s.docker.exitCode = exitCode
s.docker.fn = fn
- s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
s.docker.api = api
t.logWriter.Write(dockerLog(1, "foo\n"))
t.logWriter.Close()
}
- s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
api := &ArvTestClient{Container: rec}
kc := &KeepTestClient{}
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
+ c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
err := cr.SetupMounts()
c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
+ c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
c.Check(err, IsNil)
s.docker.fn = fn
- s.docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
+ s.docker.ImageRemove(nil, hwImageID, dockertypes.ImageRemoveOptions{})
api = &ArvTestClient{Container: rec}
kc := &KeepTestClient{}
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'tmp' for stdout"), Equals, true)
}
func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
}`, func(t *TestDockerClient) {})
c.Check(err, NotNil)
- c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
+ c.Check(strings.Contains(err.Error(), "unsupported mount kind 'collection' for stdout"), Equals, true)
}
func (s *TestSuite) TestFullRunWithAPI(c *C) {
arvlog.bytesLogged += lineSize
arvlog.logThrottleBytesSoFar += lineSize
- arvlog.logThrottleLinesSoFar += 1
+ arvlog.logThrottleLinesSoFar++
if arvlog.bytesLogged > crunchLimitLogBytesPerJob {
message = fmt.Sprintf("%s Exceeded log limit %d bytes (crunch_limit_log_bytes_per_job). Log will be truncated.",
count int
}
-func (this *TestTimestamper) Timestamp(t time.Time) string {
- this.count += 1
- t, err := time.ParseInLocation(time.RFC3339Nano, fmt.Sprintf("2015-12-29T15:51:45.%09dZ", this.count), t.Location())
+func (stamper *TestTimestamper) Timestamp(t time.Time) string {
+ stamper.count++
+ t, err := time.ParseInLocation(time.RFC3339Nano, fmt.Sprintf("2015-12-29T15:51:45.%09dZ", stamper.count), t.Location())
if err != nil {
panic(err)
}
"git.arvados.org/arvados.git/lib/controller/api"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
)
func (cq *Queue) Get(uuid string) (arvados.Container, bool) {
cq.mtx.Lock()
defer cq.mtx.Unlock()
- if ctr, ok := cq.current[uuid]; !ok {
+ ctr, ok := cq.current[uuid]
+ if !ok {
return arvados.Container{}, false
- } else {
- return ctr.Container, true
}
+ return ctr.Container, true
}
// Entries returns all cache entries, keyed by container UUID.
check "gopkg.in/check.v1"
)
+// LoadTestKey returns a public/private ssh keypair, read from the files
+// identified by the path of the private key.
func LoadTestKey(c *check.C, fnm string) (ssh.PublicKey, ssh.Signer) {
rawpubkey, err := ioutil.ReadFile(fnm + ".pub")
c.Assert(err, check.IsNil)
"postgresql",
"postgresql-contrib",
"python3-dev",
+ "python3-venv",
+ "python3-virtualenv",
"r-base",
"r-cran-testthat",
"r-cran-devtools",
"r-cran-roxygen2",
"r-cran-xml",
"sudo",
- "python3-virtualenv",
- "python3-venv",
"wget",
"xvfb",
"zlib1g-dev",
DataDirectory string
LogFile string
}
- if pg_lsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
+ if pgLsclusters, err2 := exec.Command("pg_lsclusters", "--no-header").CombinedOutput(); err2 != nil {
err = fmt.Errorf("pg_lsclusters: %s", err2)
return 1
- } else if pgclusters := strings.Split(strings.TrimSpace(string(pg_lsclusters)), "\n"); len(pgclusters) != 1 {
+ } else if pgclusters := strings.Split(strings.TrimSpace(string(pgLsclusters)), "\n"); len(pgclusters) != 1 {
logger.Warnf("pg_lsclusters returned %d postgresql clusters -- skipping postgresql initdb/startup, hope that's ok", len(pgclusters))
} else if _, err = fmt.Sscanf(pgclusters[0], "%s %s %d %s %s %s %s", &pgc.Version, &pgc.Cluster, &pgc.Port, &pgc.Status, &pgc.Owner, &pgc.DataDirectory, &pgc.LogFile); err != nil {
err = fmt.Errorf("error parsing pg_lsclusters output: %s", err)
"io"
"log"
"net/http"
+ // pprof is only imported to register its HTTP handlers
_ "net/http/pprof"
"os"
//
// SPDX-License-Identifier: Apache-2.0
-// package service provides a cmd.Handler that brings up a system service.
+// Package service provides a cmd.Handler that brings up a system service.
package service
import (
return 0
}
-const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
-
func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.FieldLogger) (arvados.URL, error) {
svc, ok := svcs.Map()[prog]
if !ok {
var _ = check.Suite(&Suite{})
type Suite struct{}
+type key int
+
+const (
+ contextKey key = iota
+)
func (*Suite) TestCommand(c *check.C) {
cf, err := ioutil.TempFile("", "cmd_test.")
defer cancel()
cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
- c.Check(ctx.Value("foo"), check.Equals, "bar")
+ c.Check(ctx.Value(contextKey), check.Equals, "bar")
c.Check(token, check.Equals, "abcde")
return &testHandler{ctx: ctx, healthCheck: healthCheck}
})
- cmd.(*command).ctx = context.WithValue(ctx, "foo", "bar")
+ cmd.(*command).ctx = context.WithValue(ctx, contextKey, "bar")
done := make(chan bool)
var stdin, stdout, stderr bytes.Buffer
key, cert := cluster.TLS.Key, cluster.TLS.Certificate
if !strings.HasPrefix(key, "file://") || !strings.HasPrefix(cert, "file://") {
- return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified as file://...")
+ return nil, errors.New("cannot use TLS certificate: TLS.Key and TLS.Certificate must be specified with a 'file://' prefix")
}
key, cert = key[7:], cert[7:]
private$REST$create(file, self$uuid)
newTreeBranch$setCollection(self)
+ newTreeBranch
})
-
- "Created"
}
else
{
collection <- arv$collections.get("uuid")
```
+Be aware that the result from `collections.get` is _not_ a
+`Collection` class. The object returned from this method lets you
+access collection fields like "name" and "description". The
+`Collection` class lets you access the files in the collection for
+reading and writing, and is described in the next section.
+
* List collections:
```{r}
collectionList <- arv$collections.list(list(list("name", "like", "Test%")))
collectionList <- arv$collections.list(list(list("name", "like", "Test%")), limit = 10, offset = 2)
-```
-```{r}
# count of total number of items (may be more than returned due to paging)
collectionList$items_available
updatedCollection <- arv$collections.update(list(name = "New name", description = "New description"), "uuid")
```
-* Create collection:
+* Create a new collection:
```{r}
newCollection <- arv$collections.create(list(name = "Example", description = "This is a test collection"))
#### Manipulating collection content
-* Create collection object:
+* Initialize a collection object:
```{r}
collection <- Collection$new(arv, "uuid")
* Write a table:
```{r}
-arvadosFile <- collection$create("myoutput.txt")
+arvadosFile <- collection$create("myoutput.txt")[[1]]
arvConnection <- arvadosFile$connection("w")
write.table(mytable, arvConnection)
arvadosFile$flush()
```
-* Write to existing file (override current content of the file):
+* Write to existing file (overwrites current content of the file):
```{r}
arvadosFile <- collection$get("location/to/my/file.cpp")
size <- arvadosSubcollection$getSizeInBytes()
```
-* Create new file in a collection:
+* Create new file in a collection (returns a vector of one or more ArvadosFile objects):
```{r}
collection$create(files)
Example:
```{r}
-mainFile <- collection$create("cpp/src/main.cpp")
+mainFile <- collection$create("cpp/src/main.cpp")[[1]]
fileList <- collection$create(c("cpp/src/main.cpp", "cpp/src/util.h"))
```
else
version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
end
+ version = version.sub("~dev", ".dev").sub("~rc", ".rc")
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.summary = "Arvados CLI tools"
s.description = "Arvados command line tools, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev@arvados.org'
+ s.email = 'packaging@arvados.org'
#s.bindir = '.'
s.licenses = ['Apache-2.0']
s.files = ["bin/arv", "bin/arv-tag", "LICENSE-2.0.txt"]
help=argparse.SUPPRESS)
parser.add_argument("--thread-count", type=int,
- default=1, help="Number of threads to use for job submit and output collection.")
+ default=4, help="Number of threads to use for job submit and output collection.")
parser.add_argument("--http-timeout", type=int,
default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
from cwltool.errors import WorkflowException
from cwltool.process import UnsupportedRequirement, shortname
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
-from cwltool.utils import aslist
+from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class
from cwltool.job import JobBase
import arvados.collection
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
runtimeContext.pull_image,
- runtimeContext.project_uuid)
+ runtimeContext.project_uuid,
+ runtimeContext.force_docker_pull,
+ runtimeContext.tmp_outdir_prefix)
network_req, _ = self.get_requirement("NetworkAccess")
if network_req:
cached_lookups = {}
cached_lookups_lock = threading.Lock()
-def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid):
+def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid,
+ force_pull, tmp_outdir_prefix):
"""Check if a Docker image is available in Keep, if not, upload it using arv-keepdocker."""
if "http://arvados.org/cwl#dockerCollectionPDH" in dockerRequirement:
if not images:
# Fetch Docker image if necessary.
try:
- cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image)
+ cwltool.docker.DockerCommandLineJob.get_image(dockerRequirement, pull_image,
+ force_pull, tmp_outdir_prefix)
except OSError as e:
raise WorkflowException("While trying to get Docker image '%s', failed to execute 'docker': %s" % (dockerRequirement["dockerImageId"], e))
from cwltool.load_tool import fetch_document, resolve_and_validate_document
from cwltool.process import shortname
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class
from cwltool.context import LoadingContext
import ruamel.yaml as yaml
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
from .perf import Perf
from .pathmapper import NoFollowPathMapper
-from .task_queue import TaskQueue
+from cwltool.task_queue import TaskQueue
from .context import ArvLoadingContext, ArvRuntimeContext
from ._version import __version__
from schema_salad.sourceline import SourceLine
from arvados.errors import ApiError
-from cwltool.pathmapper import PathMapper, MapperEnt, abspath, adjustFileObjs, adjustDirObjs
+from cwltool.pathmapper import PathMapper, MapperEnt
+from cwltool.utils import adjustFileObjs, adjustDirObjs
+from cwltool.stdfsaccess import abspath
from cwltool.workflow import WorkflowException
from .http import http_to_keep
from cwltool.process import (scandeps, UnsupportedRequirement, normalizeFilesDirs,
shortname, Process, fill_in_defaults)
from cwltool.load_tool import fetch_document
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
-from cwltool.utils import aslist
+from cwltool.utils import aslist, adjustFileObjs, adjustDirObjs, visit_class
from cwltool.builder import substitute
from cwltool.pack import pack
from cwltool.update import INTERNAL_VERSION
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
- arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
else:
- arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__}, True, arvrunner.project_uuid)
+ arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": "arvados/jobs:"+__version__},
+ True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
elif isinstance(tool, cwltool.workflow.Workflow):
for s in tool.steps:
upload_docker(arvrunner, s.embedded_tool)
if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
v["secondaryFiles"] = merged_map[cur_id].secondaryFiles[v["location"]]
if v.get("class") == "DockerRequirement":
- v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True, arvrunner.project_uuid)
+ v["http://arvados.org/cwl#dockerCollectionPDH"] = arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, v, True,
+ arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
for l in v:
visit(v[l], cur_id)
if isinstance(v, list):
"""Determine if the right arvados/jobs image version is available. If not, try to pull and upload it."""
try:
- return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ return arvados_cwl.arvdocker.arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid,
+ arvrunner.runtimeContext.force_docker_pull,
+ arvrunner.runtimeContext.tmp_outdir_prefix)
except Exception as e:
raise Exception("Docker image %s is not available\n%s" % (img, e) )
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
-from builtins import object
-
-import queue
-import threading
-import logging
-
-logger = logging.getLogger('arvados.cwl-runner')
-
-class TaskQueue(object):
- def __init__(self, lock, thread_count):
- self.thread_count = thread_count
- self.task_queue = queue.Queue(maxsize=self.thread_count)
- self.task_queue_threads = []
- self.lock = lock
- self.in_flight = 0
- self.error = None
-
- for r in range(0, self.thread_count):
- t = threading.Thread(target=self.task_queue_func)
- self.task_queue_threads.append(t)
- t.start()
-
- def task_queue_func(self):
- while True:
- task = self.task_queue.get()
- if task is None:
- return
- try:
- task()
- except Exception as e:
- logger.exception("Unhandled exception running task")
- self.error = e
-
- with self.lock:
- self.in_flight -= 1
-
- def add(self, task, unlock, check_done):
- if self.thread_count > 1:
- with self.lock:
- self.in_flight += 1
- else:
- task()
- return
-
- while True:
- try:
- unlock.release()
- if check_done.is_set():
- return
- self.task_queue.put(task, block=True, timeout=3)
- return
- except queue.Full:
- pass
- finally:
- unlock.acquire()
-
-
- def drain(self):
- try:
- # Drain queue
- while not self.task_queue.empty():
- self.task_queue.get(True, .1)
- except queue.Empty:
- pass
-
- def join(self):
- for t in self.task_queue_threads:
- self.task_queue.put(None)
- for t in self.task_queue_threads:
- t.join()
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
def get_version(setup_dir, module):
env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version(SETUP_DIR, "arvados_cwl"))
fpm_depends+=(nodejs)
case "$TARGET" in
- debian9 | ubuntu1604)
+ ubuntu1604)
fpm_depends+=(libcurl3-gnutls)
;;
debian* | ubuntu*)
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from builtins import str
-from builtins import next
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-import os
-
-SETUP_DIR = os.path.dirname(__file__) or '.'
-
-def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../python")
- else:
- getver = SETUP_DIR
- return getver
-
-class EggInfoFromGit(egg_info):
- """Tag the build with git commit timestamp.
-
- If a build tag has already been set (e.g., "egg_info -b", building
- from source package), leave it alone.
- """
- def git_latest_tag(self):
- gittags = subprocess.check_output(['git', 'tag', '-l']).split()
- gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
- return str(next(iter(gittags)).decode('utf-8'))
-
- def git_timestamp_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', choose_version_from()]).strip()
- return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
- def tags(self):
- if self.tag_build is None:
- self.tag_build = self.git_latest_tag() + self.git_timestamp_tag()
- return egg_info.tags(self)
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200807132242',
+ 'cwltool==3.0.20201121085451',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
export ARVADOS_API_HOST=localhost:8000
export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados-arvbox/superuser_token)
if test -n "$build" ; then
/usr/src/arvados/build/build-dev-docker-jobs-image.sh
. /usr/src/arvados/build/run-library.sh
TMPHERE=\$(pwd)
cd /usr/src/arvados
+
+ # This defines python_sdk_version and cwl_runner_version with python-style
+ # package suffixes (.dev/rc)
calculate_python_sdk_cwl_package_versions
+
cd \$TMPHERE
set -u
r["Clusters"][inputs.this_cluster_id] = {"RemoteClusters": remoteClusters};
if (r["Clusters"][inputs.this_cluster_id]) {
r["Clusters"][inputs.this_cluster_id]["Login"] = {"LoginCluster": inputs.cluster_ids[0]};
+ r["Clusters"][inputs.this_cluster_id]["Users"] = {"AutoAdminFirstUser": false};
}
return JSON.stringify(r);
}
arguments:
- shellQuote: false
valueFrom: |
- docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados-arvbox
docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
$(inputs.arvbox_bin.path) sv restart api
$(inputs.arvbox_bin.path) sv restart controller
$(inputs.arvbox_bin.path) restart $(inputs.arvbox_mode)
fi
$(inputs.arvbox_bin.path) status > status.txt
- $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
+ $(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token > superuser_token.txt
stubs.keep_client = keep_client2
stubs.docker_images = {
"arvados/jobs:"+arvados_cwl.__version__: [("zzzzz-4zz18-zzzzzzzzzzzzzd3", "")],
- "debian:8": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
+ "debian:buster-slim": [("zzzzz-4zz18-zzzzzzzzzzzzzd4", "")],
"arvados/jobs:123": [("zzzzz-4zz18-zzzzzzzzzzzzzd5", "")],
"arvados/jobs:latest": [("zzzzz-4zz18-zzzzzzzzzzzzzd6", "")],
}
'state': 'Committed',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'name': 'submit_wf.cwl',
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--disable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["command"] = [
'arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--disable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container["use_existing"] = False
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=stop',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
"--output-name="+output_name, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", "--debug",
"--storage-classes=foo", '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
stubs.expect_container_request_uuid + '\n')
self.assertEqual(exited, 0)
- @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("cwltool.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
@stubs
make_output.assert_called_with(u'Output of submit_wf.cwl', ['foo'], '', 'zzzzz-4zz18-zzzzzzzzzzzzzzzz')
self.assertEqual(exited, 0)
- @mock.patch("arvados_cwl.task_queue.TaskQueue")
+ @mock.patch("cwltool.task_queue.TaskQueue")
@mock.patch("arvados_cwl.arvworkflow.ArvadosWorkflow.job")
@mock.patch("arvados_cwl.executor.ArvCwlExecutor.make_output_collection")
@stubs
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
"--intermediate-output-ttl=3600",
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
"--trash-intermediate",
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
"--output-tags="+output_tags, '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
'container_image': '999999999999999999999999999999d3+99',
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow/expect_arvworkflow.cwl#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
],
'requirements': [
{
- 'dockerPull': 'debian:8',
+ 'dockerPull': 'debian:buster-slim',
'class': 'DockerRequirement',
"http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
}
'container_image': "999999999999999999999999999999d3+99",
'command': ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json'],
'cwd': '/var/spool/cwl',
expect_container["owner_uuid"] = project_uuid
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- "--eval-timeout=20", "--thread-count=1",
+ "--eval-timeout=20", "--thread-count=4",
'--enable-reuse', "--collection-cache-size=256", '--debug',
'--on-error=continue',
'--project-uuid='+project_uuid,
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=60.0', '--thread-count=1',
+ '--eval-timeout=60.0', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=256",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
expect_container = copy.deepcopy(stubs.expect_container_spec)
expect_container["command"] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=500",
'--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
}
expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
'--no-log-timestamps', '--disable-validate', '--disable-color',
- '--eval-timeout=20', '--thread-count=1',
+ '--eval-timeout=20', '--thread-count=4',
'--enable-reuse', "--collection-cache-size=512", '--debug', '--on-error=continue',
'/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
"--disable-validate",
"--disable-color",
"--eval-timeout=20",
- '--thread-count=1',
+ '--thread-count=4',
"--enable-reuse",
"--collection-cache-size=256",
'--debug',
"hints": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8",
+ "dockerPull": "debian:buster-slim",
"http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
},
{
import os
import threading
-from arvados_cwl.task_queue import TaskQueue
+from cwltool.task_queue import TaskQueue
def success_task():
pass
cwlVersion: v1.0
requirements:
- class: DockerRequirement
- dockerPull: debian:8
+ dockerPull: debian:buster-slim
inputs:
- id: x
type: File
cwlVersion: v1.0
requirements:
- class: DockerRequirement
- dockerPull: debian:8
+ dockerPull: debian:buster-slim
inputs:
- id: x
type: File
requirements:
InlineJavascriptRequirement: {}
DockerRequirement:
- dockerPull: debian:stretch-slim
+ dockerPull: debian:buster-slim
inputs:
d: Directory
outputs:
type: string
outputs: []
requirements:
- - {class: DockerRequirement, dockerPull: 'debian:8'}
+ - {class: DockerRequirement, dockerPull: 'debian:buster-slim'}
"requirements": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8",
+ "dockerPull": "debian:buster-slim",
"http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
}
]
"requirements": [
{
"class": "DockerRequirement",
- "dockerPull": "debian:8",
+ "dockerPull": "debian:buster-slim",
"http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
}
]
"cwltool:Secrets":
secrets: [pw]
DockerRequirement:
- dockerPull: debian:8
+ dockerPull: debian:buster-slim
inputs:
pw: string
outputs:
- class: CommandLineTool
requirements:
- class: DockerRequirement
- dockerPull: debian:8
+ dockerPull: debian:buster-slim
'http://arvados.org/cwl#dockerCollectionPDH': 999999999999999999999999999999d4+99
inputs:
- id: '#submit_tool.cwl/x'
# (This dockerfile file must be located in the arvados/sdk/ directory because
# of the docker build root.)
-FROM debian:9
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
defaultRequestID string
}
-// The default http.Client used by a Client with Insecure==true and
-// Client==nil.
+// InsecureHTTPClient is the default http.Client used by a Client with
+// Insecure==true and Client==nil.
var InsecureHTTPClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true}}}
-// The default http.Client used by a Client otherwise.
+// DefaultSecureClient is the default http.Client used by a Client otherwise.
var DefaultSecureClient = &http.Client{}
// NewClientFromConfig creates a new Client that uses the endpoints in
return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params)
}
+// RequestAndDecodeContext does the same as RequestAndDecode, but with a context
func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error {
if body, ok := body.(io.Closer); ok {
// Ensure body is closed even if we error out early
}
}
}
- if cc, ok := sc.Clusters[clusterID]; !ok {
+ cc, ok := sc.Clusters[clusterID]
+ if !ok {
return nil, fmt.Errorf("cluster %q is not configured", clusterID)
- } else {
- cc.ClusterID = clusterID
- return &cc, nil
}
+ cc.ClusterID = clusterID
+ return &cc, nil
}
type WebDAVCacheConfig struct {
MaxKeepBlobBuffers int
MaxRequestAmplification int
MaxRequestSize int
- RailsSessionSecretToken string
RequestTimeout Duration
SendTimeout Duration
WebsocketClientEventQueue int
FinishedAt *time.Time `json:"finished_at"` // nil if not yet finished
}
-// Container is an arvados#container resource.
+// ContainerRequest is an arvados#container_request resource.
type ContainerRequest struct {
UUID string `json:"uuid"`
OwnerUUID string `json:"owner_uuid"`
ContainerStateCancelled = ContainerState("Cancelled")
)
-// ContainerState is a string corresponding to a valid Container state.
+// ContainerRequestState is a string corresponding to a valid Container Request state.
type ContainerRequestState string
const (
// it fails, we'll try again next time.
close(done)
return nil
- } else {
- // In sync mode, we proceed regardless of
- // whether another flush is in progress: It
- // can't finish before we do, because we hold
- // fn's lock until we finish our own writes.
}
+ // In sync mode, we proceed regardless of
+ // whether another flush is in progress: It
+ // can't finish before we do, because we hold
+ // fn's lock until we finish our own writes.
seg.flushing = done
offsets = append(offsets, len(block))
if len(refs) == 1 {
Properties map[string]interface{} `json:"properties"`
}
-// UserList is an arvados#userList resource.
+// LinkList is an arvados#linkList resource.
type LinkList struct {
Items []Link `json:"items"`
ItemsAvailable int `json:"items_available"`
defaultHTTPClientMtx sync.Mutex
)
-// Indicates an error that was returned by the API server.
+// APIServerError contains an error that was returned by the API server.
type APIServerError struct {
// Address of server returning error, of the form "host:port".
ServerAddress string
return s == "1" || s == "yes" || s == "true"
}
-// Helper type so we don't have to write out 'map[string]interface{}' every time.
+// Dict is a helper type so we don't have to write out 'map[string]interface{}' every time.
type Dict map[string]interface{}
-// Information about how to contact the Arvados server
+// ArvadosClient contains information about how to contact the Arvados server
type ArvadosClient struct {
// https
Scheme string
Scheme: scheme,
Host: c.ApiServer}
- if resourceType != API_DISCOVERY_RESOURCE {
+ if resourceType != ApiDiscoveryResource {
u.Path = "/arvados/v1"
}
return c.Call("DELETE", resource, uuid, "", parameters, output)
}
-// Modify attributes of a resource. See Call for argument descriptions.
+// Update attributes of a resource. See Call for argument descriptions.
func (c *ArvadosClient) Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error) {
return c.Call("PUT", resourceType, uuid, "", parameters, output)
}
return c.Call("GET", resource, "", "", parameters, output)
}
-const API_DISCOVERY_RESOURCE = "discovery/v1/apis/arvados/v1/rest"
+const ApiDiscoveryResource = "discovery/v1/apis/arvados/v1/rest"
// Discovery returns the value of the given parameter in the discovery
// document. Returns a non-nil error if the discovery document cannot
func (c *ArvadosClient) Discovery(parameter string) (value interface{}, err error) {
if len(c.DiscoveryDoc) == 0 {
c.DiscoveryDoc = make(Dict)
- err = c.Call("GET", API_DISCOVERY_RESOURCE, "", "", nil, &c.DiscoveryDoc)
+ err = c.Call("GET", ApiDiscoveryResource, "", "", nil, &c.DiscoveryDoc)
if err != nil {
return nil, err
}
return value, ErrInvalidArgument
}
-func (ac *ArvadosClient) httpClient() *http.Client {
- if ac.Client != nil {
- return ac.Client
+func (c *ArvadosClient) httpClient() *http.Client {
+ if c.Client != nil {
+ return c.Client
}
- c := &defaultSecureHTTPClient
- if ac.ApiInsecure {
- c = &defaultInsecureHTTPClient
+ cl := &defaultSecureHTTPClient
+ if c.ApiInsecure {
+ cl = &defaultInsecureHTTPClient
}
- if *c == nil {
+ if *cl == nil {
defaultHTTPClientMtx.Lock()
defer defaultHTTPClientMtx.Unlock()
- *c = &http.Client{Transport: &http.Transport{
- TLSClientConfig: MakeTLSConfig(ac.ApiInsecure)}}
+ *cl = &http.Client{Transport: &http.Transport{
+ TLSClientConfig: MakeTLSConfig(c.ApiInsecure)}}
}
- return *c
+ return *cl
}
return url.URL{Scheme: "https", Host: "apistub.example.com"}
}
func (as *APIStub) ConfigGet(ctx context.Context) (json.RawMessage, error) {
- as.appendCall(as.ConfigGet, ctx, nil)
+ as.appendCall(ctx, as.ConfigGet, nil)
return nil, as.Error
}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
- as.appendCall(as.Login, ctx, options)
+ as.appendCall(ctx, as.Login, options)
return arvados.LoginResponse{}, as.Error
}
func (as *APIStub) Logout(ctx context.Context, options arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- as.appendCall(as.Logout, ctx, options)
+ as.appendCall(ctx, as.Logout, options)
return arvados.LogoutResponse{}, as.Error
}
func (as *APIStub) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionCreate, ctx, options)
+ as.appendCall(ctx, as.CollectionCreate, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) CollectionUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionUpdate, ctx, options)
+ as.appendCall(ctx, as.CollectionUpdate, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) CollectionGet(ctx context.Context, options arvados.GetOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionGet, ctx, options)
+ as.appendCall(ctx, as.CollectionGet, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) CollectionList(ctx context.Context, options arvados.ListOptions) (arvados.CollectionList, error) {
- as.appendCall(as.CollectionList, ctx, options)
+ as.appendCall(ctx, as.CollectionList, options)
return arvados.CollectionList{}, as.Error
}
func (as *APIStub) CollectionProvenance(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
- as.appendCall(as.CollectionProvenance, ctx, options)
+ as.appendCall(ctx, as.CollectionProvenance, options)
return nil, as.Error
}
func (as *APIStub) CollectionUsedBy(ctx context.Context, options arvados.GetOptions) (map[string]interface{}, error) {
- as.appendCall(as.CollectionUsedBy, ctx, options)
+ as.appendCall(ctx, as.CollectionUsedBy, options)
return nil, as.Error
}
func (as *APIStub) CollectionDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionDelete, ctx, options)
+ as.appendCall(ctx, as.CollectionDelete, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) CollectionTrash(ctx context.Context, options arvados.DeleteOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionTrash, ctx, options)
+ as.appendCall(ctx, as.CollectionTrash, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) CollectionUntrash(ctx context.Context, options arvados.UntrashOptions) (arvados.Collection, error) {
- as.appendCall(as.CollectionUntrash, ctx, options)
+ as.appendCall(ctx, as.CollectionUntrash, options)
return arvados.Collection{}, as.Error
}
func (as *APIStub) ContainerCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerCreate, ctx, options)
+ as.appendCall(ctx, as.ContainerCreate, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) ContainerUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerUpdate, ctx, options)
+ as.appendCall(ctx, as.ContainerUpdate, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerGet, ctx, options)
+ as.appendCall(ctx, as.ContainerGet, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) ContainerList(ctx context.Context, options arvados.ListOptions) (arvados.ContainerList, error) {
- as.appendCall(as.ContainerList, ctx, options)
+ as.appendCall(ctx, as.ContainerList, options)
return arvados.ContainerList{}, as.Error
}
func (as *APIStub) ContainerDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerDelete, ctx, options)
+ as.appendCall(ctx, as.ContainerDelete, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) ContainerLock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerLock, ctx, options)
+ as.appendCall(ctx, as.ContainerLock, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) ContainerUnlock(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
- as.appendCall(as.ContainerUnlock, ctx, options)
+ as.appendCall(ctx, as.ContainerUnlock, options)
return arvados.Container{}, as.Error
}
func (as *APIStub) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
- as.appendCall(as.SpecimenCreate, ctx, options)
+ as.appendCall(ctx, as.SpecimenCreate, options)
return arvados.Specimen{}, as.Error
}
func (as *APIStub) SpecimenUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Specimen, error) {
- as.appendCall(as.SpecimenUpdate, ctx, options)
+ as.appendCall(ctx, as.SpecimenUpdate, options)
return arvados.Specimen{}, as.Error
}
func (as *APIStub) SpecimenGet(ctx context.Context, options arvados.GetOptions) (arvados.Specimen, error) {
- as.appendCall(as.SpecimenGet, ctx, options)
+ as.appendCall(ctx, as.SpecimenGet, options)
return arvados.Specimen{}, as.Error
}
func (as *APIStub) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
- as.appendCall(as.SpecimenList, ctx, options)
+ as.appendCall(ctx, as.SpecimenList, options)
return arvados.SpecimenList{}, as.Error
}
func (as *APIStub) SpecimenDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Specimen, error) {
- as.appendCall(as.SpecimenDelete, ctx, options)
+ as.appendCall(ctx, as.SpecimenDelete, options)
return arvados.Specimen{}, as.Error
}
func (as *APIStub) UserCreate(ctx context.Context, options arvados.CreateOptions) (arvados.User, error) {
- as.appendCall(as.UserCreate, ctx, options)
+ as.appendCall(ctx, as.UserCreate, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.User, error) {
- as.appendCall(as.UserUpdate, ctx, options)
+ as.appendCall(ctx, as.UserUpdate, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserUpdateUUID(ctx context.Context, options arvados.UpdateUUIDOptions) (arvados.User, error) {
- as.appendCall(as.UserUpdateUUID, ctx, options)
+ as.appendCall(ctx, as.UserUpdateUUID, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserActivate(ctx context.Context, options arvados.UserActivateOptions) (arvados.User, error) {
- as.appendCall(as.UserActivate, ctx, options)
+ as.appendCall(ctx, as.UserActivate, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserSetup(ctx context.Context, options arvados.UserSetupOptions) (map[string]interface{}, error) {
- as.appendCall(as.UserSetup, ctx, options)
+ as.appendCall(ctx, as.UserSetup, options)
return nil, as.Error
}
func (as *APIStub) UserUnsetup(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- as.appendCall(as.UserUnsetup, ctx, options)
+ as.appendCall(ctx, as.UserUnsetup, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserGet(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- as.appendCall(as.UserGet, ctx, options)
+ as.appendCall(ctx, as.UserGet, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserGetCurrent(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- as.appendCall(as.UserGetCurrent, ctx, options)
+ as.appendCall(ctx, as.UserGetCurrent, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserGetSystem(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- as.appendCall(as.UserGetSystem, ctx, options)
+ as.appendCall(ctx, as.UserGetSystem, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserList(ctx context.Context, options arvados.ListOptions) (arvados.UserList, error) {
- as.appendCall(as.UserList, ctx, options)
+ as.appendCall(ctx, as.UserList, options)
return arvados.UserList{}, as.Error
}
func (as *APIStub) UserDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.User, error) {
- as.appendCall(as.UserDelete, ctx, options)
+ as.appendCall(ctx, as.UserDelete, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserMerge(ctx context.Context, options arvados.UserMergeOptions) (arvados.User, error) {
- as.appendCall(as.UserMerge, ctx, options)
+ as.appendCall(ctx, as.UserMerge, options)
return arvados.User{}, as.Error
}
func (as *APIStub) UserBatchUpdate(ctx context.Context, options arvados.UserBatchUpdateOptions) (arvados.UserList, error) {
- as.appendCall(as.UserBatchUpdate, ctx, options)
+ as.appendCall(ctx, as.UserBatchUpdate, options)
return arvados.UserList{}, as.Error
}
func (as *APIStub) UserAuthenticate(ctx context.Context, options arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
- as.appendCall(as.UserAuthenticate, ctx, options)
+ as.appendCall(ctx, as.UserAuthenticate, options)
return arvados.APIClientAuthorization{}, as.Error
}
func (as *APIStub) APIClientAuthorizationCurrent(ctx context.Context, options arvados.GetOptions) (arvados.APIClientAuthorization, error) {
- as.appendCall(as.APIClientAuthorizationCurrent, ctx, options)
+ as.appendCall(ctx, as.APIClientAuthorizationCurrent, options)
return arvados.APIClientAuthorization{}, as.Error
}
-func (as *APIStub) appendCall(method interface{}, ctx context.Context, options interface{}) {
+func (as *APIStub) appendCall(ctx context.Context, method interface{}, options interface{}) {
as.mtx.Lock()
defer as.mtx.Unlock()
as.calls = append(as.calls, APIStubCall{method, ctx, options})
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/jmoiron/sqlx"
+ // sqlx needs lib/pq to talk to PostgreSQL
_ "github.com/lib/pq"
"gopkg.in/check.v1"
)
RunningContainerUUID = "zzzzz-dz642-runningcontainr"
- CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerUUID = "zzzzz-dz642-compltcontainer"
+ CompletedContainerRequestUUID = "zzzzz-xvhdp-cr4completedctr"
+ CompletedContainerRequestUUID2 = "zzzzz-xvhdp-cr4completedcr2"
+
+ CompletedDiagnosticsContainerRequest1UUID = "zzzzz-xvhdp-diagnostics0001"
+ CompletedDiagnosticsContainerRequest2UUID = "zzzzz-xvhdp-diagnostics0002"
+ CompletedDiagnosticsContainer1UUID = "zzzzz-dz642-diagcompreq0001"
+ CompletedDiagnosticsContainer2UUID = "zzzzz-dz642-diagcompreq0002"
+ DiagnosticsContainerRequest1LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog1"
+ DiagnosticsContainerRequest2LogCollectionUUID = "zzzzz-4zz18-diagcompreqlog2"
+
+ CompletedDiagnosticsHasher1ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0001"
+ CompletedDiagnosticsHasher2ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0002"
+ CompletedDiagnosticsHasher3ContainerRequestUUID = "zzzzz-xvhdp-diag1hasher0003"
+ CompletedDiagnosticsHasher1ContainerUUID = "zzzzz-dz642-diagcomphasher1"
+ CompletedDiagnosticsHasher2ContainerUUID = "zzzzz-dz642-diagcomphasher2"
+ CompletedDiagnosticsHasher3ContainerUUID = "zzzzz-dz642-diagcomphasher3"
+
+ Hasher1LogCollectionUUID = "zzzzz-4zz18-dlogcollhash001"
+ Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
+ Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
TestVMUUID = "zzzzz-2x53u-382brsig8rp3064"
CollectionWithUniqueWordsUUID = "zzzzz-4zz18-mnt690klmb51aud"
+
+ LogCollectionUUID = "zzzzz-4zz18-logcollection01"
+ LogCollectionUUID2 = "zzzzz-4zz18-logcollection02"
)
// PathologicalManifest : A valid manifest designed to test
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvadostest
+
+import (
+ "crypto/rand"
+ "crypto/rsa"
+ "encoding/base64"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "time"
+
+ "gopkg.in/check.v1"
+ "gopkg.in/square/go-jose.v2"
+)
+
+type OIDCProvider struct {
+ // expected token request
+ ValidCode string
+ ValidClientID string
+ ValidClientSecret string
+ // desired response from token endpoint
+ AuthEmail string
+ AuthEmailVerified bool
+ AuthName string
+
+ PeopleAPIResponse map[string]interface{}
+
+ key *rsa.PrivateKey
+ Issuer *httptest.Server
+ PeopleAPI *httptest.Server
+ c *check.C
+}
+
+func NewOIDCProvider(c *check.C) *OIDCProvider {
+ p := &OIDCProvider{c: c}
+ var err error
+ p.key, err = rsa.GenerateKey(rand.Reader, 2048)
+ c.Assert(err, check.IsNil)
+ p.Issuer = httptest.NewServer(http.HandlerFunc(p.serveOIDC))
+ p.PeopleAPI = httptest.NewServer(http.HandlerFunc(p.servePeopleAPI))
+ return p
+}
+
+func (p *OIDCProvider) ValidAccessToken() string {
+ return p.fakeToken([]byte("fake access token"))
+}
+
+func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) {
+ req.ParseForm()
+ p.c.Logf("serveOIDC: got req: %s %s %s", req.Method, req.URL, req.Form)
+ w.Header().Set("Content-Type", "application/json")
+ switch req.URL.Path {
+ case "/.well-known/openid-configuration":
+ json.NewEncoder(w).Encode(map[string]interface{}{
+ "issuer": p.Issuer.URL,
+ "authorization_endpoint": p.Issuer.URL + "/auth",
+ "token_endpoint": p.Issuer.URL + "/token",
+ "jwks_uri": p.Issuer.URL + "/jwks",
+ "userinfo_endpoint": p.Issuer.URL + "/userinfo",
+ })
+ case "/token":
+ var clientID, clientSecret string
+ auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
+ authsplit := strings.Split(string(auth), ":")
+ if len(authsplit) == 2 {
+ clientID, _ = url.QueryUnescape(authsplit[0])
+ clientSecret, _ = url.QueryUnescape(authsplit[1])
+ }
+ if clientID != p.ValidClientID || clientSecret != p.ValidClientSecret {
+ p.c.Logf("OIDCProvider: expected (%q, %q) got (%q, %q)", p.ValidClientID, p.ValidClientSecret, clientID, clientSecret)
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
+ if req.Form.Get("code") != p.ValidCode || p.ValidCode == "" {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ idToken, _ := json.Marshal(map[string]interface{}{
+ "iss": p.Issuer.URL,
+ "aud": []string{clientID},
+ "sub": "fake-user-id",
+ "exp": time.Now().UTC().Add(time.Minute).Unix(),
+ "iat": time.Now().UTC().Unix(),
+ "nonce": "fake-nonce",
+ "email": p.AuthEmail,
+ "email_verified": p.AuthEmailVerified,
+ "name": p.AuthName,
+ "alt_verified": true, // for custom claim tests
+ "alt_email": "alt_email@example.com", // for custom claim tests
+ "alt_username": "desired-username", // for custom claim tests
+ })
+ json.NewEncoder(w).Encode(struct {
+ AccessToken string `json:"access_token"`
+ TokenType string `json:"token_type"`
+ RefreshToken string `json:"refresh_token"`
+ ExpiresIn int32 `json:"expires_in"`
+ IDToken string `json:"id_token"`
+ }{
+ AccessToken: p.ValidAccessToken(),
+ TokenType: "Bearer",
+ RefreshToken: "test-refresh-token",
+ ExpiresIn: 30,
+ IDToken: p.fakeToken(idToken),
+ })
+ case "/jwks":
+ json.NewEncoder(w).Encode(jose.JSONWebKeySet{
+ Keys: []jose.JSONWebKey{
+ {Key: p.key.Public(), Algorithm: string(jose.RS256), KeyID: ""},
+ },
+ })
+ case "/auth":
+ w.WriteHeader(http.StatusInternalServerError)
+ case "/userinfo":
+ if authhdr := req.Header.Get("Authorization"); strings.TrimPrefix(authhdr, "Bearer ") != p.ValidAccessToken() {
+ p.c.Logf("OIDCProvider: bad auth %q", authhdr)
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ json.NewEncoder(w).Encode(map[string]interface{}{
+ "sub": "fake-user-id",
+ "name": p.AuthName,
+ "given_name": p.AuthName,
+ "family_name": "",
+ "alt_username": "desired-username",
+ "email": p.AuthEmail,
+ "email_verified": p.AuthEmailVerified,
+ })
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+}
+
+func (p *OIDCProvider) servePeopleAPI(w http.ResponseWriter, req *http.Request) {
+ req.ParseForm()
+ p.c.Logf("servePeopleAPI: got req: %s %s %s", req.Method, req.URL, req.Form)
+ w.Header().Set("Content-Type", "application/json")
+ switch req.URL.Path {
+ case "/v1/people/me":
+ if f := req.Form.Get("personFields"); f != "emailAddresses,names" {
+ w.WriteHeader(http.StatusBadRequest)
+ break
+ }
+ json.NewEncoder(w).Encode(p.PeopleAPIResponse)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+}
+
+func (p *OIDCProvider) fakeToken(payload []byte) string {
+ signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: p.key}, nil)
+ if err != nil {
+ p.c.Error(err)
+ return ""
+ }
+ object, err := signer.Sign(payload)
+ if err != nil {
+ p.c.Error(err)
+ return ""
+ }
+ t, err := object.CompactSerialize()
+ if err != nil {
+ p.c.Error(err)
+ return ""
+ }
+ p.c.Logf("fakeToken(%q) == %q", payload, t)
+ return t
+}
a.Tokens = append(a.Tokens, string(token))
}
-// LoadTokensFromHTTPRequestBody() loads credentials from the request
+// LoadTokensFromHTTPRequestBody loads credentials from the request
// body.
//
// This is separate from LoadTokensFromHTTPRequest() because it's not
var LocatorPattern = regexp.MustCompile(
"^[0-9a-fA-F]{32}\\+[0-9]+(\\+[A-Z][A-Za-z0-9@_-]*)*$")
-// Stores a Block Locator Digest compactly, up to 128 bits.
-// Can be used as a map key.
+// BlockDigest stores a Block Locator Digest compactly, up to 128 bits. Can be
+// used as a map key.
type BlockDigest struct {
H uint64
L uint64
return fmt.Sprintf("%s+%d", w.Digest.String(), w.Size)
}
-// Will create a new BlockDigest unless an error is encountered.
+// FromString creates a new BlockDigest unless an error is encountered.
func FromString(s string) (dig BlockDigest, err error) {
if len(s) != 32 {
err = fmt.Errorf("Block digest should be exactly 32 characters but this one is %d: %s", len(s), s)
func getStackTrace() string {
buf := make([]byte, 1000)
- bytes_written := runtime.Stack(buf, false)
- return "Stack Trace:\n" + string(buf[:bytes_written])
+ bytesWritten := runtime.Stack(buf, false)
+ return "Stack Trace:\n" + string(buf[:bytesWritten])
}
func expectEqual(t *testing.T, actual interface{}, expected interface{}) {
package blockdigest
-// Just used for testing when we need some distinct BlockDigests
+// MakeTestBlockDigest is used for testing with distinct BlockDigests
func MakeTestBlockDigest(i int) BlockDigest {
return BlockDigest{L: uint64(i)}
}
// Reads from the underlying reader, update the hashing function, and
// pass the results through. Returns BadChecksum (instead of EOF) on
// the last read if the checksum doesn't match.
-func (this HashCheckingReader) Read(p []byte) (n int, err error) {
- n, err = this.Reader.Read(p)
+func (hcr HashCheckingReader) Read(p []byte) (n int, err error) {
+ n, err = hcr.Reader.Read(p)
if n > 0 {
- this.Hash.Write(p[:n])
+ hcr.Hash.Write(p[:n])
}
if err == io.EOF {
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
err = BadChecksum
}
}
return n, err
}
-// WriteTo writes the entire contents of this.Reader to dest. Returns
+// WriteTo writes the entire contents of hcr.Reader to dest. Returns
// BadChecksum if writing is successful but the checksum doesn't
// match.
-func (this HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
- if writeto, ok := this.Reader.(io.WriterTo); ok {
- written, err = writeto.WriteTo(io.MultiWriter(dest, this.Hash))
+func (hcr HashCheckingReader) WriteTo(dest io.Writer) (written int64, err error) {
+ if writeto, ok := hcr.Reader.(io.WriterTo); ok {
+ written, err = writeto.WriteTo(io.MultiWriter(dest, hcr.Hash))
} else {
- written, err = io.Copy(io.MultiWriter(dest, this.Hash), this.Reader)
+ written, err = io.Copy(io.MultiWriter(dest, hcr.Hash), hcr.Reader)
}
if err != nil {
return written, err
}
- sum := this.Hash.Sum(nil)
- if fmt.Sprintf("%x", sum) != this.Check {
+ sum := hcr.Hash.Sum(nil)
+ if fmt.Sprintf("%x", sum) != hcr.Check {
return written, BadChecksum
}
// Close reads all remaining data from the underlying Reader and
// returns BadChecksum if the checksum doesn't match. It also closes
// the underlying Reader if it implements io.ReadCloser.
-func (this HashCheckingReader) Close() (err error) {
- _, err = io.Copy(this.Hash, this.Reader)
+func (hcr HashCheckingReader) Close() (err error) {
+ _, err = io.Copy(hcr.Hash, hcr.Reader)
- if closer, ok := this.Reader.(io.Closer); ok {
+ if closer, ok := hcr.Reader.(io.Closer); ok {
closeErr := closer.Close()
if err == nil {
err = closeErr
if err != nil {
return err
}
- if fmt.Sprintf("%x", this.Hash.Sum(nil)) != this.Check {
+ if fmt.Sprintf("%x", hcr.Hash.Sum(nil)) != hcr.Check {
return BadChecksum
}
return nil
//
// SPDX-License-Identifier: Apache-2.0
-/* Provides low-level Get/Put primitives for accessing Arvados Keep blocks. */
+// Package keepclient provides low-level Get/Put primitives for accessing
+// Arvados Keep blocks.
package keepclient
import (
"git.arvados.org/arvados.git/sdk/go/httpserver"
)
-// A Keep "block" is 64MB.
+// BLOCKSIZE defines the length of a Keep "block", which is 64MB.
const BLOCKSIZE = 64 * 1024 * 1024
var (
// ErrIncompleteIndex is returned when the Index response does not end with a new empty line
var ErrIncompleteIndex = errors.New("Got incomplete index")
-const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
-const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
+const XKeepDesiredReplicas = "X-Keep-Desired-Replicas"
+const XKeepReplicasStored = "X-Keep-Replicas-Stored"
type HTTPClient interface {
Do(*http.Request) (*http.Response, error)
}
-// Information about Arvados and Keep servers.
+// KeepClient holds information about Arvados and Keep servers.
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
}
}
-// Put a block given the block hash, a reader, and the number of bytes
+// PutHR puts a block given the block hash, a reader, and the number of bytes
// to read from the reader (which must be between 0 and BLOCKSIZE).
//
// Returns the locator for the written block, the number of replicas
//
// If the block hash and data size are known, PutHR is more efficient.
func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error) {
- if buffer, err := ioutil.ReadAll(r); err != nil {
+ buffer, err := ioutil.ReadAll(r)
+ if err != nil {
return "", 0, err
- } else {
- return kc.PutB(buffer)
}
+ return kc.PutB(buffer)
}
func (kc *KeepClient) getOrHead(method string, locator string, header http.Header) (io.ReadCloser, int64, string, http.Header, error) {
var errs []string
- tries_remaining := 1 + kc.Retries
+ triesRemaining := 1 + kc.Retries
serversToTry := kc.getSortedRoots(locator)
var retryList []string
- for tries_remaining > 0 {
- tries_remaining -= 1
+ for triesRemaining > 0 {
+ triesRemaining--
retryList = nil
for _, host := range serversToTry {
return loc, nil
}
-// Get() retrieves a block, given a locator. Returns a reader, the
+// Get retrieves a block, given a locator. Returns a reader, the
// expected data length, the URL the block is being fetched from, and
// an error.
//
return rdr, size, url, err
}
-// ReadAt() retrieves a portion of block from the cache if it's
+// ReadAt retrieves a portion of block from the cache if it's
// present, otherwise from the network.
func (kc *KeepClient) ReadAt(locator string, p []byte, off int) (int, error) {
return kc.cache().ReadAt(kc, locator, p, off)
}
-// Ask() verifies that a block with the given hash is available and
+// Ask verifies that a block with the given hash is available and
// readable, according to at least one Keep service. Unlike Get, it
// does not retrieve the data or verify that the data content matches
// the hash specified by the locator.
return bytes.NewReader(respBody[0 : len(respBody)-1]), nil
}
-// LocalRoots() returns the map of local (i.e., disk and proxy) Keep
+// LocalRoots returns the map of local (i.e., disk and proxy) Keep
// services: uuid -> baseURI.
func (kc *KeepClient) LocalRoots() map[string]string {
kc.discoverServices()
return kc.localRoots
}
-// GatewayRoots() returns the map of Keep remote gateway services:
+// GatewayRoots returns the map of Keep remote gateway services:
// uuid -> baseURI.
func (kc *KeepClient) GatewayRoots() map[string]string {
kc.discoverServices()
return kc.gatewayRoots
}
-// WritableLocalRoots() returns the map of writable local Keep services:
+// WritableLocalRoots returns the map of writable local Keep services:
// uuid -> baseURI.
func (kc *KeepClient) WritableLocalRoots() map[string]string {
kc.discoverServices()
type StubPutHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
expectBody string
expectStorageClass string
handled chan string
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
- sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
kc, _ := MakeKeepClient(arv)
reader, writer := io.Pipe()
- upload_status := make(chan uploadStatus)
+ uploadStatusChan := make(chan uploadStatus)
- f(kc, ks.url, reader, writer, upload_status)
+ f(kc, ks.url, reader, writer, uploadStatusChan)
}
func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
kc.StorageClasses = []string{"hot"}
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
+ go kc.uploadToKeepServer(url, st.expectPath, reader, uploadStatusChan, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
<-st.handled
- status := <-upload_status
+ status := <-uploadStatusChan
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
}
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
+ func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID())
<-st.handled
- status := <-upload_status
+ status := <-uploadStatusChan
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
}
fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
if fh.count == 0 {
resp.WriteHeader(500)
- fh.count += 1
+ fh.count++
fh.handled <- fmt.Sprintf("http://%s", req.Host)
} else {
fh.successhandler.ServeHTTP(resp, req)
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
+ writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
+ go kc.uploadToKeepServer(url, hash, reader, uploadStatusChan, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
<-st.handled
- status := <-upload_status
+ status := <-uploadStatusChan
c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
c.Check(status.statusCode, Equals, 500)
})
func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
- for i := 0; i < n; i += 1 {
+ for i := 0; i < n; i++ {
ks[i] = RunFakeKeepServer(st)
}
type StubGetHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
httpStatus int
body []byte
}
func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
- sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+ sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
resp.WriteHeader(sgh.httpStatus)
resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
resp.Write(sgh.body)
handled chan string
}
-func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
resp.Write([]byte("bar"))
- this.handled <- fmt.Sprintf("http://%s", req.Host)
+ h.handled <- fmt.Sprintf("http://%s", req.Host)
}
func (s *StandaloneSuite) TestChecksum(c *C) {
c.Check(n, Equals, int64(3))
c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
- read_content, err2 := ioutil.ReadAll(r)
+ readContent, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
- c.Check(read_content, DeepEquals, content)
+ c.Check(readContent, DeepEquals, content)
}
func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
c.Check(n, Equals, int64(len(content)))
c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
- read_content, err2 := ioutil.ReadAll(r)
+ readContent, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
- c.Check(read_content, DeepEquals, content)
+ c.Check(readContent, DeepEquals, content)
}
{
n, url2, err := kc.Ask(hash)
handled chan string
}
-func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("X-Keep-Replicas-Stored", "2")
- this.handled <- fmt.Sprintf("http://%s", req.Host)
+ h.handled <- fmt.Sprintf("http://%s", req.Host)
}
func (s *StandaloneSuite) TestPutProxy(c *C) {
return fmt.Sprintf("https://%x.svc/", i)
}
-func FakeSvcUuid(i uint64) string {
+func FakeSvcUUID(i uint64) string {
return fmt.Sprintf("zzzzz-bi6l4-%015x", i)
}
func FakeServiceRoots(n uint64) map[string]string {
sr := map[string]string{}
for i := uint64(0); i < n; i++ {
- sr[FakeSvcUuid(i)] = FakeSvcRoot(i)
+ sr[FakeSvcUUID(i)] = FakeSvcRoot(i)
}
return sr
}
fakeroots := FakeServiceRoots(16)
// These reference probe orders are explained further in
// ../../python/tests/test_keep_client.py:
- expected_orders := []string{
+ expectedOrders := []string{
"3eab2d5fc9681074",
"097dba52e648f1c3",
"c5b4e023f8a7d691",
"9d81c02e76a3bf54",
}
- for h, expected_order := range expected_orders {
+ for h, expectedOrder := range expectedOrders {
hash := Md5String(fmt.Sprintf("%064x", h))
roots := NewRootSorter(fakeroots, hash).GetSortedRoots()
- for i, svc_id_s := range strings.Split(expected_order, "") {
- svc_id, err := strconv.ParseUint(svc_id_s, 16, 64)
+ for i, svcIDs := range strings.Split(expectedOrder, "") {
+ svcID, err := strconv.ParseUint(svcIDs, 16, 64)
c.Assert(err, Equals, nil)
- c.Check(roots[i], Equals, FakeSvcRoot(svc_id))
+ c.Check(roots[i], Equals, FakeSvcRoot(svcID))
}
}
}
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
)
-// Function used to emit debug messages. The easiest way to enable
+// DebugPrintf emits debug messages. The easiest way to enable
// keepclient debug messages in your application is to assign
// log.Printf to DebugPrintf.
var DebugPrintf = func(string, ...interface{}) {}
}
type uploadStatus struct {
- err error
- url string
- statusCode int
- replicas_stored int
- response string
+ err error
+ url string
+ statusCode int
+ replicasStored int
+ response string
}
-func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
- upload_status chan<- uploadStatus, expectedLength int64, reqid string) {
+func (kc *KeepClient) uploadToKeepServer(host string, hash string, body io.Reader,
+ uploadStatusChan chan<- uploadStatus, expectedLength int64, reqid string) {
var req *http.Request
var err error
var url = fmt.Sprintf("%s/%s", host, hash)
if req, err = http.NewRequest("PUT", url, nil); err != nil {
DebugPrintf("DEBUG: [%s] Error creating request PUT %v error: %v", reqid, url, err.Error())
- upload_status <- uploadStatus{err, url, 0, 0, ""}
+ uploadStatusChan <- uploadStatus{err, url, 0, 0, ""}
return
}
}
req.Header.Add("X-Request-Id", reqid)
- req.Header.Add("Authorization", "OAuth2 "+this.Arvados.ApiToken)
+ req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken)
req.Header.Add("Content-Type", "application/octet-stream")
- req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
- if len(this.StorageClasses) > 0 {
- req.Header.Add("X-Keep-Storage-Classes", strings.Join(this.StorageClasses, ", "))
+ req.Header.Add(XKeepDesiredReplicas, fmt.Sprint(kc.Want_replicas))
+ if len(kc.StorageClasses) > 0 {
+ req.Header.Add("X-Keep-Storage-Classes", strings.Join(kc.StorageClasses, ", "))
}
var resp *http.Response
- if resp, err = this.httpClient().Do(req); err != nil {
+ if resp, err = kc.httpClient().Do(req); err != nil {
DebugPrintf("DEBUG: [%s] Upload failed %v error: %v", reqid, url, err.Error())
- upload_status <- uploadStatus{err, url, 0, 0, err.Error()}
+ uploadStatusChan <- uploadStatus{err, url, 0, 0, err.Error()}
return
}
rep := 1
- if xr := resp.Header.Get(X_Keep_Replicas_Stored); xr != "" {
+ if xr := resp.Header.Get(XKeepReplicasStored); xr != "" {
fmt.Sscanf(xr, "%d", &rep)
}
response := strings.TrimSpace(string(respbody))
if err2 != nil && err2 != io.EOF {
DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, err2.Error(), response)
- upload_status <- uploadStatus{err2, url, resp.StatusCode, rep, response}
+ uploadStatusChan <- uploadStatus{err2, url, resp.StatusCode, rep, response}
} else if resp.StatusCode == http.StatusOK {
DebugPrintf("DEBUG: [%s] Upload %v success", reqid, url)
- upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
+ uploadStatusChan <- uploadStatus{nil, url, resp.StatusCode, rep, response}
} else {
if resp.StatusCode >= 300 && response == "" {
response = resp.Status
}
DebugPrintf("DEBUG: [%s] Upload %v error: %v response: %v", reqid, url, resp.StatusCode, response)
- upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
+ uploadStatusChan <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
}
}
-func (this *KeepClient) putReplicas(
+func (kc *KeepClient) putReplicas(
hash string,
getReader func() io.Reader,
expectedLength int64) (locator string, replicas int, err error) {
- reqid := this.getRequestID()
+ reqid := kc.getRequestID()
// Calculate the ordering for uploading to servers
- sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
+ sv := NewRootSorter(kc.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
nextServer := 0
active := 0
// Used to communicate status from the upload goroutines
- upload_status := make(chan uploadStatus)
+ uploadStatusChan := make(chan uploadStatus)
defer func() {
// Wait for any abandoned uploads (e.g., we started
// two uploads and the first replied with replicas=2)
// to finish before closing the status channel.
go func() {
for active > 0 {
- <-upload_status
+ <-uploadStatusChan
}
- close(upload_status)
+ close(uploadStatusChan)
}()
}()
replicasDone := 0
- replicasTodo := this.Want_replicas
+ replicasTodo := kc.Want_replicas
- replicasPerThread := this.replicasPerService
+ replicasPerThread := kc.replicasPerService
if replicasPerThread < 1 {
// unlimited or unknown
replicasPerThread = replicasTodo
}
- retriesRemaining := 1 + this.Retries
+ retriesRemaining := 1 + kc.Retries
var retryServers []string
lastError := make(map[string]string)
for retriesRemaining > 0 {
- retriesRemaining -= 1
+ retriesRemaining--
nextServer = 0
retryServers = []string{}
for replicasTodo > 0 {
// Start some upload requests
if nextServer < len(sv) {
DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
- go this.uploadToKeepServer(sv[nextServer], hash, getReader(), upload_status, expectedLength, reqid)
- nextServer += 1
- active += 1
+ go kc.uploadToKeepServer(sv[nextServer], hash, getReader(), uploadStatusChan, expectedLength, reqid)
+ nextServer++
+ active++
} else {
if active == 0 && retriesRemaining == 0 {
msg := "Could not write sufficient replicas: "
// Now wait for something to happen.
if active > 0 {
- status := <-upload_status
- active -= 1
+ status := <-uploadStatusChan
+ active--
if status.statusCode == 200 {
// good news!
- replicasDone += status.replicas_stored
- replicasTodo -= status.replicas_stored
+ replicasDone += status.replicasStored
+ replicasTodo -= status.replicasStored
locator = status.response
delete(lastError, status.url)
} else {
Name string
}
-// Represents a single line from a manifest.
+// ManifestStream represents a single line from a manifest.
type ManifestStream struct {
StreamName string
Blocks []string
return ch
}
-func firstBlock(offsets []uint64, range_start uint64) int {
- // range_start/block_start is the inclusive lower bound
- // range_end/block_end is the exclusive upper bound
+func firstBlock(offsets []uint64, rangeStart uint64) int {
+ // rangeStart/blockStart is the inclusive lower bound
+ // rangeEnd/blockEnd is the exclusive upper bound
hi := len(offsets) - 1
var lo int
i := ((hi + lo) / 2)
- block_start := offsets[i]
- block_end := offsets[i+1]
+ blockStart := offsets[i]
+ blockEnd := offsets[i+1]
// perform a binary search for the first block
- // assumes that all of the blocks are contiguous, so range_start is guaranteed
+ // assumes that all of the blocks are contiguous, so rangeStart is guaranteed
// to either fall into the range of a block or be outside the block range entirely
- for !(range_start >= block_start && range_start < block_end) {
+ for !(rangeStart >= blockStart && rangeStart < blockEnd) {
if lo == i {
// must be out of range, fail
return -1
}
- if range_start > block_start {
+ if rangeStart > blockStart {
lo = i
} else {
hi = i
}
i = ((hi + lo) / 2)
- block_start = offsets[i]
- block_end = offsets[i+1]
+ blockStart = offsets[i]
+ blockEnd = offsets[i+1]
}
return i
}
}
sort.Strings(sortedfiles)
- stream_tokens := []string{EscapeName(name)}
+ streamTokens := []string{EscapeName(name)}
blocks := make(map[blockdigest.BlockDigest]int64)
var streamoffset int64
for _, segment := range stream[streamfile] {
b, _ := ParseBlockLocator(segment.Locator)
if _, ok := blocks[b.Digest]; !ok {
- stream_tokens = append(stream_tokens, segment.Locator)
+ streamTokens = append(streamTokens, segment.Locator)
blocks[b.Digest] = streamoffset
streamoffset += int64(b.Size)
}
}
}
- if len(stream_tokens) == 1 {
- stream_tokens = append(stream_tokens, "d41d8cd98f00b204e9800998ecf8427e+0")
+ if len(streamTokens) == 1 {
+ streamTokens = append(streamTokens, "d41d8cd98f00b204e9800998ecf8427e+0")
}
for _, streamfile := range sortedfiles {
// Add in file segments
- span_start := int64(-1)
- span_end := int64(0)
+ spanStart := int64(-1)
+ spanEnd := int64(0)
fout := EscapeName(streamfile)
for _, segment := range stream[streamfile] {
// Collapse adjacent segments
b, _ := ParseBlockLocator(segment.Locator)
streamoffset = blocks[b.Digest] + int64(segment.Offset)
- if span_start == -1 {
- span_start = streamoffset
- span_end = streamoffset + int64(segment.Len)
+ if spanStart == -1 {
+ spanStart = streamoffset
+ spanEnd = streamoffset + int64(segment.Len)
} else {
- if streamoffset == span_end {
- span_end += int64(segment.Len)
+ if streamoffset == spanEnd {
+ spanEnd += int64(segment.Len)
} else {
- stream_tokens = append(stream_tokens, fmt.Sprintf("%d:%d:%s", span_start, span_end-span_start, fout))
- span_start = streamoffset
- span_end = streamoffset + int64(segment.Len)
+ streamTokens = append(streamTokens, fmt.Sprintf("%d:%d:%s", spanStart, spanEnd-spanStart, fout))
+ spanStart = streamoffset
+ spanEnd = streamoffset + int64(segment.Len)
}
}
}
- if span_start != -1 {
- stream_tokens = append(stream_tokens, fmt.Sprintf("%d:%d:%s", span_start, span_end-span_start, fout))
+ if spanStart != -1 {
+ streamTokens = append(streamTokens, fmt.Sprintf("%d:%d:%s", spanStart, spanEnd-spanStart, fout))
}
if len(stream[streamfile]) == 0 {
- stream_tokens = append(stream_tokens, fmt.Sprintf("0:0:%s", fout))
+ streamTokens = append(streamTokens, fmt.Sprintf("0:0:%s", fout))
}
}
- return strings.Join(stream_tokens, " ") + "\n"
+ return strings.Join(streamTokens, " ") + "\n"
}
func (m segmentedManifest) manifestTextForPath(srcpath, relocate string) string {
filesegs, okfile := stream[filename]
if okfile {
newstream := make(segmentedStream)
- relocate_stream, relocate_filename := splitPath(relocate)
- if relocate_filename == "" {
- relocate_filename = filename
+ relocateStream, relocateFilename := splitPath(relocate)
+ if relocateFilename == "" {
+ relocateFilename = filename
}
- newstream[relocate_filename] = filesegs
- return newstream.normalizedText(relocate_stream)
+ newstream[relocateFilename] = filesegs
+ return newstream.normalizedText(relocateStream)
}
}
return ch
}
+// BlockIterWithDuplicates iterates over the block locators of a manifest.
+//
// Blocks may appear multiple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
1. Add this Arvados repository to your sources list::
- deb http://apt.arvados.org/ stretch main
+ deb http://apt.arvados.org/ buster main
2. Update your package list.
-3. Install the ``python-arvados-python-client`` package.
+3. Install the ``python3-arvados-python-client`` package.
Configuration
-------------
import os
import re
import socket
+import sys
import time
import types
RETRY_DELAY_BACKOFF = 2
RETRY_COUNT = 2
+if sys.version_info >= (3,):
+ httplib2.SSLHandshakeError = None
+
class OrderedJsonModel(apiclient.model.JsonModel):
"""Model class for JSON that preserves the contents' order.
class KeepWriterThread(threading.Thread):
- TaskFailed = RuntimeError()
+ class TaskFailed(RuntimeError): pass
def __init__(self, queue, data, data_hash, timeout=None):
super(KeepClient.KeepWriterThread, self).__init__()
try:
locator, copies = self.do_task(service, service_root)
except Exception as e:
- if e is not self.TaskFailed:
+ if not isinstance(e, self.TaskFailed):
_logger.exception("Exception in KeepWriterThread")
self.queue.write_fail(service)
else:
self.data_hash,
result['status_code'],
result['body'])
- raise self.TaskFailed
+ raise self.TaskFailed()
_logger.debug("KeepWriterThread %s succeeded %s+%i %s",
str(threading.current_thread()),
offset = c['offset'] + len(c['items'])
return items
+def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"]
+ other_filters = kwargs.get("filters", [])
+
+ if "select" in kwargs and "uuid" not in kwargs["select"]:
+ kwargs["select"].append("uuid")
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+
def ca_certs_path(fallback=httplib2.CA_CERTS):
"""Return the path of the best available CA certs source.
import time
import os
import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ return getver
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
def get_version(setup_dir, module):
env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version(SETUP_DIR, "arvados"))
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-from setuptools.command.egg_info import egg_info
-import subprocess
-import time
-
-class EggInfoFromGit(egg_info):
- """Tag the build with git commit timestamp.
-
- If a build tag has already been set (e.g., "egg_info -b", building
- from source package), leave it alone.
- """
- def git_latest_tag(self):
- gittags = subprocess.check_output(['git', 'tag', '-l']).split()
- gittags.sort(key=lambda s: [int(u) for u in s.split(b'.')],reverse=True)
- return str(next(iter(gittags)).decode('utf-8'))
-
- def git_timestamp_tag(self):
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', '.']).strip()
- return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo)))
-
- def tags(self):
- if self.tag_build is None:
- self.tag_build = self.git_latest_tag()+self.git_timestamp_tag()
- return egg_info.tags(self)
in the 'fed_migrate' input parameter.
# Create arvbox containers fedbox(1,2,3) for the federation
-$ cwltool arvbox-make-federation.cwl --arvbox_base ~/.arvbox > fed.json
+$ cwltool --preserve-environment=SSH_AUTH_SOCK arvbox-make-federation.cwl --arvbox_base ~/.arvbox > fed.json
# Configure containers and run tests
-$ cwltool fed-migrate.cwl fed.json
+$ cwltool --preserve-environment=SSH_AUTH_SOCK fed-migrate.cwl fed.json
CWL for running the test is generated using cwl-ex:
- arguments:
- arvbox
- cat
- - /var/lib/arvados/superuser_token
+ - /var/lib/arvados-arvbox/superuser_token
class: CommandLineTool
cwlVersion: v1.0
id: '#superuser_tok'
ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path)
- cat /var/lib/arvados/vm-uuid)
+ cat /var/lib/arvados-arvbox/vm-uuid)
ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat
- /var/lib/arvados/superuser_token)
+ /var/lib/arvados-arvbox/superuser_token)
while ! curl --fail --insecure --silent -H
"Authorization: Bearer $ARVADOS_API_TOKEN"
while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
-ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid)
-ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
+ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/vm-uuid)
+ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token)
while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID >/dev/null ; do sleep 3 ; done
>>>
report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate)
return supertok, report
-}
\ No newline at end of file
+}
envDef:
ARVBOX_CONTAINER: "$(inputs.container)"
InlineJavascriptRequirement: {}
-arguments: [arvbox, cat, /var/lib/arvados/superuser_token]
+arguments: [arvbox, cat, /var/lib/arvados-arvbox/superuser_token]
my_api_host = None
_cached_config = {}
_cached_db_config = {}
+_already_used_port = {}
def find_server_pid(PID_PATH, wait=10):
now = time.time()
would take care of the races, and this wouldn't be needed at all.
"""
- sock = socket.socket()
- sock.bind(('0.0.0.0', 0))
- port = sock.getsockname()[1]
- sock.close()
- return port
+ global _already_used_port
+ while True:
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ port = sock.getsockname()[1]
+ sock.close()
+ if port not in _already_used_port:
+ _already_used_port[port] = True
+ return port
def _wait_until_port_listens(port, timeout=10, warn=True):
"""Wait for a process to start listening on the given port.
"SystemRootToken": auth_token('system_user'),
"API": {
"RequestTimeout": "30s",
- "RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
},
"Login": {
"SSO": {
import unittest
import arvados
+import arvados.util
class MkdirDashPTest(unittest.TestCase):
def setUp(self):
def test_failure(self):
with self.assertRaises(arvados.errors.CommandFailedError):
arvados.util.run_command(['false'])
+
+class KeysetTestHelper:
+ def __init__(self, expect):
+ self.n = 0
+ self.expect = expect
+
+ def fn(self, **kwargs):
+ if self.expect[self.n][0] != kwargs:
+ raise Exception("Didn't match %s != %s" % (self.expect[self.n][0], kwargs))
+ return self
+
+ def execute(self, num_retries):
+ self.n += 1
+ return self.expect[self.n-1][1]
+
+class KeysetListAllTestCase(unittest.TestCase):
+ def test_empty(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [])
+
+ def test_oneitem(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "1"], ["uuid", ">", "1"]]},
+ {"items": []}
+ ],[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}])
+
+ def test_onepage2(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+ def test_onepage3(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "3"], ["uuid", "!=", "3"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "3", "uuid": "3"}])
+
+
+ def test_twopage(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "3"}, {"created_at": "4", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "4"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "3", "uuid": "3"},
+ {"created_at": "4", "uuid": "4"}
+ ])
+
+ def test_repeated_key(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "3"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "3"]]},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "2", "uuid": "4"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", "=", "2"], ["uuid", ">", "4"]]},
+ {"items": []}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">", "2"]]},
+ {"items": [{"created_at": "3", "uuid": "5"}, {"created_at": "4", "uuid": "6"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "4"], ["uuid", "!=", "6"]]},
+ {"items": []}
+ ],
+ ])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"},
+ {"created_at": "2", "uuid": "2"},
+ {"created_at": "2", "uuid": "3"},
+ {"created_at": "2", "uuid": "4"},
+ {"created_at": "3", "uuid": "5"},
+ {"created_at": "4", "uuid": "6"}
+ ])
+
+ def test_onepage_withfilter(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["foo", ">", "bar"]]},
+ {"items": [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at asc", "uuid asc"], "filters": [["created_at", ">=", "2"], ["uuid", "!=", "2"], ["foo", ">", "bar"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, filters=[["foo", ">", "bar"]]))
+ self.assertEqual(ls, [{"created_at": "1", "uuid": "1"}, {"created_at": "2", "uuid": "2"}])
+
+
+ def test_onepage_desc(self):
+ ks = KeysetTestHelper([[
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": []},
+ {"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]}
+ ], [
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]},
+ {"items": []}
+ ]])
+
+ ls = list(arvados.util.keyset_list_all(ks.fn, ascending=False))
+ self.assertEqual(ls, [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}])
else
version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
end
+ version = version.sub("~dev", ".dev").sub("~rc", ".rc")
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.summary = "Arvados client library"
s.description = "Arvados client library, git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev@curoverse.com'
+ s.email = 'packaging@arvados.org'
s.licenses = ['Apache-2.0']
s.files = ["lib/arvados.rb", "lib/arvados/google_api_client.rb",
"lib/arvados/collection.rb", "lib/arvados/keep.rb",
class TransactionFailedError < StandardError
end
- @@config = nil
@@debuglevel = 0
class << self
attr_accessor :debuglevel
@arvados_api_version = opts[:api_version] || 'v1'
- @arvados_api_host = opts[:api_host] ||
- config['ARVADOS_API_HOST'] or
- raise "#{$0}: no :api_host or ENV[ARVADOS_API_HOST] provided."
- @arvados_api_token = opts[:api_token] ||
- config['ARVADOS_API_TOKEN'] or
- raise "#{$0}: no :api_token or ENV[ARVADOS_API_TOKEN] provided."
+ @config = nil
+ [[:api_host, 'ARVADOS_API_HOST'],
+ [:api_token, 'ARVADOS_API_TOKEN']].each do |op, en|
+ if opts[op]
+ config[en] = opts[op]
+ end
+ if !config[en]
+ raise "#{$0}: no :#{op} or ENV[#{en}] provided."
+ end
+ end
if (opts[:suppress_ssl_warnings] or
%w(1 true yes).index(config['ARVADOS_API_HOST_INSECURE'].
# result looks like Arvados::A26949680::Job.
namespace_class.const_set classname, klass
- self.class.class_eval do
- define_method classname.underscore do
- klass
- end
+ self.define_singleton_method classname.underscore do
+ klass
end
end
end
def client
@client ||= Google::APIClient.
- new(:host => @arvados_api_host,
+ new(:host => config["ARVADOS_API_HOST"],
:application_name => @application_name,
:application_version => @application_version.to_s)
end
end
def config(config_file_path="~/.config/arvados/settings.conf")
- return @@config if @@config
+ return @config if @config
# Initialize config settings with environment variables.
config = {}
# Note: If we start using additional configuration settings from
# this file in the future, we might have to read the file anyway
# instead of returning here.
- return (@@config = config)
+ return (@config = config)
end
begin
debuglog "Ignoring error reading #{config_file_path}: #{e}", 0
end
- @@config = config
+ @config = config
end
class Model
:parameters => parameters,
:body_object => body,
:headers => {
- :authorization => 'OAuth2 '+arvados.config['ARVADOS_API_TOKEN']
+ :authorization => 'Bearer '+arvados.config['ARVADOS_API_TOKEN']
})
resp = JSON.parse result.body, :symbolize_names => true
if resp[:errors]
if params[pname].is_a?(Boolean)
return params[pname]
else
- logger.warn "Warning: received non-boolean parameter '#{pname}' on #{self.class.inspect}."
+ logger.warn "Warning: received non-boolean value #{params[pname].inspect} for boolean parameter #{pname} on #{self.class.inspect}, treating as false."
end
end
false
if @objects.respond_to? :except
list[:items_available] = @objects.
except(:limit).except(:offset).
- distinct.count(:id)
+ count(@distinct ? :id : '*')
end
when 'none'
else
# Make sure params[key] is either true or false -- not a
# string, not nil, etc.
if not params.include?(key)
- params[key] = info[:default]
+ params[key] = info[:default] || false
elsif [false, 'false', '0', 0].include? params[key]
params[key] = false
elsif [true, 'true', '1', 1].include? params[key]
val.is_a?(String) && (attr == 'uuid' || attr == 'api_token')
}
end
- @objects = model_class.where('user_id=?', current_user.id)
+ if current_api_client_authorization.andand.api_token != Rails.configuration.SystemRootToken
+ @objects = model_class.where('user_id=?', current_user.id)
+ end
if wanted_scopes.compact.any?
# We can't filter on scopes effectively using AR/postgres.
# Instead we get the entire result set, do our own filtering on
def find_object_by_uuid
uuid_param = params[:uuid] || params[:id]
- if (uuid_param != current_api_client_authorization.andand.uuid and
- not Thread.current[:api_client].andand.is_trusted)
+ if (uuid_param != current_api_client_authorization.andand.uuid &&
+ !Thread.current[:api_client].andand.is_trusted)
return forbidden
end
@limit = 1
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Include collections whose is_trashed attribute is true."
+ type: 'boolean', required: false, default: false, description: "Include collections whose is_trashed attribute is true.",
},
include_old_versions: {
- type: 'boolean', required: false, description: "Include past collection versions."
+ type: 'boolean', required: false, default: false, description: "Include past collection versions.",
},
})
end
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Show collection even if its is_trashed attribute is true."
+ type: 'boolean', required: false, default: false, description: "Show collection even if its is_trashed attribute is true.",
},
include_old_versions: {
- type: 'boolean', required: false, description: "Include past collection versions."
+ type: 'boolean', required: false, default: true, description: "Include past collection versions.",
},
})
end
super
end
- def find_objects_for_index
- opts = {}
- if params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name)
- opts.update({include_trash: true})
- end
- if params[:include_old_versions] || @include_old_versions
- opts.update({include_old_versions: true})
+ def update
+ # preserve_version should be disabled unless explicitly asked otherwise.
+ if !resource_attrs[:preserve_version]
+ resource_attrs[:preserve_version] = false
end
+ super
+ end
+
+ def find_objects_for_index
+ opts = {
+ include_trash: params[:include_trash] || ['destroy', 'trash', 'untrash'].include?(action_name),
+ include_old_versions: params[:include_old_versions] || false,
+ }
@objects = Collection.readable_by(*@read_users, opts) if !opts.empty?
super
end
def find_object_by_uuid
- if params[:include_old_versions].nil?
- @include_old_versions = true
- else
- @include_old_versions = params[:include_old_versions]
- end
-
if loc = Keep::Locator.parse(params[:id])
loc.strip_hints!
- opts = {}
- opts.update({include_trash: true}) if params[:include_trash]
- opts.update({include_old_versions: @include_old_versions})
+ opts = {
+ include_trash: params[:include_trash],
+ include_old_versions: params[:include_old_versions],
+ }
# It matters which Collection object we pick because we use it to get signed_manifest_text,
# the value of which is affected by the value of trash_at.
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Include container requests whose owner project is trashed."
+ type: 'boolean', required: false, default: false, description: "Include container requests whose owner project is trashed.",
},
})
end
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Show container request even if its owner project is trashed."
+ type: 'boolean', required: false, default: false, description: "Show container request even if its owner project is trashed.",
},
})
end
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Include items whose is_trashed attribute is true."
+ type: 'boolean', required: false, default: false, description: "Include items whose is_trashed attribute is true.",
},
})
end
(super rescue {}).
merge({
include_trash: {
- type: 'boolean', required: false, description: "Show group/project even if its is_trashed attribute is true."
+ type: 'boolean', required: false, default: false, description: "Show group/project even if its is_trashed attribute is true.",
},
})
end
params = _index_requires_parameters.
merge({
uuid: {
- type: 'string', required: false, default: nil
+ type: 'string', required: false, default: nil,
},
recursive: {
- type: 'boolean', required: false, description: 'Include contents from child groups recursively.'
+ type: 'boolean', required: false, default: false, description: 'Include contents from child groups recursively.',
},
include: {
- type: 'string', required: false, description: 'Include objects referred to by listed field in "included" (only owner_uuid)'
+ type: 'string', required: false, description: 'Include objects referred to by listed field in "included" (only owner_uuid).',
+ },
+ include_old_versions: {
+ type: 'boolean', required: false, default: false, description: 'Include past collection versions.',
}
})
params.delete(:select)
type: 'boolean',
location: 'query',
default: false,
- description: 'defer permissions update'
+ description: 'defer permissions update',
}
}
)
type: 'boolean',
location: 'query',
default: false,
- description: 'defer permissions update'
+ description: 'defer permissions update',
}
}
)
@select = nil
where_conds = filter_by_owner
if klass == Collection
- @select = klass.selectable_attributes - ["manifest_text"]
+ @select = klass.selectable_attributes - ["manifest_text", "unsigned_manifest_text"]
elsif klass == Group
where_conds = where_conds.merge(group_class: "project")
end
end
end.compact
- @objects = klass.readable_by(*@read_users, {:include_trash => params[:include_trash]}).
- order(request_order).where(where_conds)
+ @objects = klass.readable_by(*@read_users, {
+ :include_trash => params[:include_trash],
+ :include_old_versions => params[:include_old_versions]
+ }).order(request_order).where(where_conds)
if params['exclude_home_project']
@objects = exclude_home @objects, klass
(super rescue {}).
merge({
find_or_create: {
- type: 'boolean', required: false, default: false
+ type: 'boolean', required: false, default: false,
},
filters: {
- type: 'array', required: false
+ type: 'array', required: false,
},
minimum_script_version: {
- type: 'string', required: false
+ type: 'string', required: false,
},
exclude_script_versions: {
- type: 'array', required: false
+ type: 'array', required: false,
},
})
end
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20200331",
+ revision: "20201210",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
end
@response = @object.setup(repo_name: full_repo_name,
- vm_uuid: params[:vm_uuid])
-
- # setup succeeded. send email to user
- if params[:send_notification_email] && !Rails.configuration.Users.UserSetupMailText.empty?
- begin
- UserNotifier.account_is_setup(@object).deliver_now
- rescue => e
- logger.warn "Failed to send email to #{@object.email}: #{e}"
- end
- end
+ vm_uuid: params[:vm_uuid],
+ send_notification_email: params[:send_notification_email])
send_json kind: "arvados#HashList", items: @response.as_api_response(nil)
end
type: 'string', required: false,
},
redirect_to_new_user: {
- type: 'boolean', required: false,
+ type: 'boolean', required: false, default: false,
},
old_user_uuid: {
type: 'string', required: false,
def self._setup_requires_parameters
{
uuid: {
- type: 'string', required: false
+ type: 'string', required: false,
},
user: {
- type: 'object', required: false
+ type: 'object', required: false,
},
repo_name: {
- type: 'string', required: false
+ type: 'string', required: false,
},
vm_uuid: {
- type: 'string', required: false
+ type: 'string', required: false,
},
send_notification_email: {
- type: 'boolean', required: false, default: false
+ type: 'boolean', required: false, default: false,
},
}
end
def self._update_requires_parameters
super.merge({
bypass_federation: {
- type: 'boolean', required: false,
+ type: 'boolean', required: false, default: false,
},
})
end
auth = nil
[params["api_token"],
params["oauth_token"],
- env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([-\/a-zA-Z0-9]+)/).andand[2],
+ env["HTTP_AUTHORIZATION"].andand.match(/(OAuth2|Bearer) ([!-~]+)/).andand[2],
*reader_tokens,
].each do |supplied|
next if !supplied
return auth
end
+ token_uuid = ''
+ secret = token
+ stored_secret = nil # ...if different from secret
+ optional = nil
+
case token[0..2]
when 'v2/'
_, token_uuid, secret, optional = token.split('/')
return auth
end
- token_uuid_prefix = token_uuid[0..4]
- if token_uuid_prefix == Rails.configuration.ClusterID
+ upstream_cluster_id = token_uuid[0..4]
+ if upstream_cluster_id == Rails.configuration.ClusterID
# Token is supposedly issued by local cluster, but if the
# token were valid, we would have been found in the database
# in the above query.
return nil
- elsif token_uuid_prefix.length != 5
+ elsif upstream_cluster_id.length != 5
# malformed
return nil
end
- # Invariant: token_uuid_prefix != Rails.configuration.ClusterID
- #
- # In other words the remaing code in this method below is the
- # case that determines whether to accept a token that was issued
- # by a remote cluster when the token absent or expired in our
- # database. To begin, we need to ask the cluster that issued
- # the token to [re]validate it.
- clnt = ApiClientAuthorization.make_http_client(uuid_prefix: token_uuid_prefix)
-
- host = remote_host(uuid_prefix: token_uuid_prefix)
- if !host
- Rails.logger.warn "remote authentication rejected: no host for #{token_uuid_prefix.inspect}"
+ else
+ # token is not a 'v2' token. It could be just the secret part
+ # ("v1 token") -- or it could be an OpenIDConnect access token,
+ # in which case either (a) the controller will have inserted a
+ # row with api_token = hmac(systemroottoken,oidctoken) before
+ # forwarding it, or (b) we'll have done that ourselves, or (c)
+ # we'll need to ask LoginCluster to validate it for us below,
+ # and then insert a local row for a faster lookup next time.
+ hmac = OpenSSL::HMAC.hexdigest('sha256', Rails.configuration.SystemRootToken, token)
+ auth = ApiClientAuthorization.
+ includes(:user, :api_client).
+ where('api_token in (?, ?) and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token, hmac).
+ first
+ if auth && auth.user
+ return auth
+ elsif !Rails.configuration.Login.LoginCluster.blank? && Rails.configuration.Login.LoginCluster != Rails.configuration.ClusterID
+ # An unrecognized non-v2 token might be an OIDC Access Token
+ # that can be verified by our login cluster in the code
+ # below. If so, we'll stuff the database with hmac instead of
+ # the real OIDC token.
+ upstream_cluster_id = Rails.configuration.Login.LoginCluster
+ stored_secret = hmac
+ else
return nil
end
+ end
+
+ # Invariant: upstream_cluster_id != Rails.configuration.ClusterID
+ #
+ # In other words the remaining code in this method decides
+ # whether to accept a token that was issued by a remote cluster
+ # when the token is absent or expired in our database. To
+ # begin, we need to ask the cluster that issued the token to
+ # [re]validate it.
+ clnt = ApiClientAuthorization.make_http_client(uuid_prefix: upstream_cluster_id)
+
+ host = remote_host(uuid_prefix: upstream_cluster_id)
+ if !host
+ Rails.logger.warn "remote authentication rejected: no host for #{upstream_cluster_id.inspect}"
+ return nil
+ end
+
+ begin
+ remote_user = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/users/current',
+ {'remote' => Rails.configuration.ClusterID},
+ {'Authorization' => 'Bearer ' + token}))
+ rescue => e
+ Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}"
+ return nil
+ end
+ # Check the response is well formed.
+ if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String)
+ Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}"
+ return nil
+ end
+
+ remote_user_prefix = remote_user['uuid'][0..4]
+
+ if token_uuid == ''
+ # Use the same UUID as the remote when caching the token.
begin
- remote_user = SafeJSON.load(
- clnt.get_content('https://' + host + '/arvados/v1/users/current',
+ remote_token = SafeJSON.load(
+ clnt.get_content('https://' + host + '/arvados/v1/api_client_authorizations/current',
{'remote' => Rails.configuration.ClusterID},
{'Authorization' => 'Bearer ' + token}))
+ token_uuid = remote_token['uuid']
+ if !token_uuid.match(HasUuid::UUID_REGEX) || token_uuid[0..4] != upstream_cluster_id
+ raise "remote cluster #{upstream_cluster_id} returned invalid token uuid #{token_uuid.inspect}"
+ end
rescue => e
- Rails.logger.warn "remote authentication with token #{token.inspect} failed: #{e}"
- return nil
- end
-
- # Check the response is well formed.
- if !remote_user.is_a?(Hash) || !remote_user['uuid'].is_a?(String)
- Rails.logger.warn "remote authentication rejected: remote_user=#{remote_user.inspect}"
- return nil
- end
-
- remote_user_prefix = remote_user['uuid'][0..4]
-
- # Clusters can only authenticate for their own users.
- if remote_user_prefix != token_uuid_prefix
- Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{token_uuid_prefix}"
+ Rails.logger.warn "error getting remote token details for #{token.inspect}: #{e}"
return nil
end
+ end
- # Invariant: remote_user_prefix == token_uuid_prefix
- # therefore: remote_user_prefix != Rails.configuration.ClusterID
+ # Clusters can only authenticate for their own users.
+ if remote_user_prefix != upstream_cluster_id
+ Rails.logger.warn "remote authentication rejected: claimed remote user #{remote_user_prefix} but token was issued by #{upstream_cluster_id}"
+ return nil
+ end
- # Add or update user and token in local database so we can
- # validate subsequent requests faster.
+ # Invariant: remote_user_prefix == upstream_cluster_id
+ # therefore: remote_user_prefix != Rails.configuration.ClusterID
- if remote_user['uuid'][-22..-1] == '-tpzed-anonymouspublic'
- # Special case: map the remote anonymous user to local anonymous user
- remote_user['uuid'] = anonymous_user_uuid
- end
+ # Add or update user and token in local database so we can
+ # validate subsequent requests faster.
- user = User.find_by_uuid(remote_user['uuid'])
+ if remote_user['uuid'][-22..-1] == '-tpzed-anonymouspublic'
+ # Special case: map the remote anonymous user to local anonymous user
+ remote_user['uuid'] = anonymous_user_uuid
+ end
- if !user
- # Create a new record for this user.
- user = User.new(uuid: remote_user['uuid'],
- is_active: false,
- is_admin: false,
- email: remote_user['email'],
- owner_uuid: system_user_uuid)
- user.set_initial_username(requested: remote_user['username'])
- end
+ user = User.find_by_uuid(remote_user['uuid'])
- # Sync user record.
- if remote_user_prefix == Rails.configuration.Login.LoginCluster
- # Remote cluster controls our user database, set is_active if
- # remote is active. If remote is not active, user will be
- # unsetup (see below).
- user.is_active = true if remote_user['is_active']
- user.is_admin = remote_user['is_admin']
- else
- if Rails.configuration.Users.NewUsersAreActive ||
- Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"]
- # Default policy is to activate users
- user.is_active = true if remote_user['is_active']
- end
- end
+ if !user
+ # Create a new record for this user.
+ user = User.new(uuid: remote_user['uuid'],
+ is_active: false,
+ is_admin: false,
+ email: remote_user['email'],
+ owner_uuid: system_user_uuid)
+ user.set_initial_username(requested: remote_user['username'])
+ end
+ # Sync user record.
+ act_as_system_user do
%w[first_name last_name email prefs].each do |attr|
user.send(attr+'=', remote_user[attr])
end
user.last_name = "from cluster #{remote_user_prefix}"
end
- act_as_system_user do
- if (user.is_active && !remote_user['is_active']) or (user.is_invited && !remote_user['is_invited'])
- # Synchronize the user's "active/invited" state state. This
- # also saves the record.
- user.unsetup
- else
- user.save!
+ user.save!
+
+ if user.is_invited && !remote_user['is_invited']
+ # Remote user is not "invited" state, they should be unsetup, which
+ # also makes them inactive.
+ user.unsetup
+ else
+ if !user.is_invited && remote_user['is_invited'] and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.AutoSetupNewUsers or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ user.setup
end
- # We will accept this token (and avoid reloading the user
- # record) for 'RemoteTokenRefresh' (default 5 minutes).
- # Possible todo:
- # Request the actual api_client_auth record from the remote
- # server in case it wants the token to expire sooner.
- auth = ApiClientAuthorization.find_or_create_by(uuid: token_uuid) do |auth|
- auth.user = user
- auth.api_client_id = 0
+ if !user.is_active && remote_user['is_active'] && user.is_invited and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ user.update_attributes!(is_active: true)
+ elsif user.is_active && !remote_user['is_active']
+ user.update_attributes!(is_active: false)
+ end
+
+ if remote_user_prefix == Rails.configuration.Login.LoginCluster and
+ user.is_active and
+ user.is_admin != remote_user['is_admin']
+ # Remote cluster controls our user database, including the
+ # admin flag.
+ user.update_attributes!(is_admin: remote_user['is_admin'])
end
- auth.update_attributes!(user: user,
- api_token: secret,
- api_client_id: 0,
- expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
- Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
end
- return auth
- else
- # token is not a 'v2' token
- auth = ApiClientAuthorization.
- includes(:user, :api_client).
- where('api_token=? and (expires_at is null or expires_at > CURRENT_TIMESTAMP)', token).
- first
- if auth && auth.user
- return auth
+
+ # We will accept this token (and avoid reloading the user
+ # record) for 'RemoteTokenRefresh' (default 5 minutes).
+ # Possible todo:
+ # Request the actual api_client_auth record from the remote
+ # server in case it wants the token to expire sooner.
+ auth = ApiClientAuthorization.find_or_create_by(uuid: token_uuid) do |auth|
+ auth.user = user
+ auth.api_client_id = 0
end
+ # If stored_secret is set, we save stored_secret in the database
+ # but return the real secret to the caller. This way, if we end
+ # up returning the auth record to the client, they see the same
+ # secret they supplied, instead of the HMAC we saved in the
+ # database.
+ stored_secret = stored_secret || secret
+ auth.update_attributes!(user: user,
+ api_token: stored_secret,
+ api_client_id: 0,
+ expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
+ Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
+ auth.api_token = secret
+ return auth
end
return nil
end
def log_update
-
super unless (saved_changes.keys - UNLOGGED_CHANGES).empty?
end
end
sql_conds = nil
user_uuids = users_list.map { |u| u.uuid }
+ all_user_uuids = []
# For details on how the trashed_groups table is constructed, see
# see db/migrate/20200501150153_permission_table.rb
exclude_trashed_records = "AND (#{sql_table}.trash_at is NULL or #{sql_table}.trash_at > statement_timestamp())"
end
+ trashed_check = ""
+ if !include_trash && sql_table != "api_client_authorizations"
+ trashed_check = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} " +
+ "where trash_at <= statement_timestamp()) #{exclude_trashed_records}"
+ end
+
if users_list.select { |u| u.is_admin }.any?
# Admin skips most permission checks, but still want to filter on trashed items.
- if !include_trash
- if sql_table != "api_client_authorizations"
- # Only include records where the owner is not trashed
- sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} "+
- "where trash_at <= statement_timestamp()) #{exclude_trashed_records}"
- end
+ if !include_trash && sql_table != "api_client_authorizations"
+ # Only include records where the owner is not trashed
+ sql_conds = trashed_check
end
else
- trashed_check = ""
- if !include_trash then
- trashed_check = "AND target_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} where trash_at <= statement_timestamp())"
- end
-
# The core of the permission check is a join against the
# materialized_permissions table to determine if the user has at
# least read permission to either the object itself or its
# A user can have can_manage access to another user, this grants
# full access to all that user's stuff. To implement that we
# need to include those other users in the permission query.
- user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: ":user_uuids", perm_level: 1}
+
+ # This was previously implemented by embedding the subquery
+ # directly into the query, but it was discovered later that this
+ # causes the Postgres query planner to do silly things because
+ # the query heuristics assumed the subquery would have a lot
+ # more rows that it does, and choose a bad merge strategy. By
+ # doing the query here and embedding the result as a constant,
+ # Postgres also knows exactly how many items there are and can
+ # choose the right query strategy.
+ #
+ # (note: you could also do this with a temporary table, but that
+ # would require all every request be wrapped in a transaction,
+ # which is not currently the case).
+
+ all_user_uuids = ActiveRecord::Base.connection.exec_query %{
+#{USER_UUIDS_SUBQUERY_TEMPLATE % {user: "'#{user_uuids.join "', '"}'", perm_level: 1}}
+},
+ 'readable_by.user_uuids'
+
+ user_uuids_subquery = ":user_uuids"
# Note: it is possible to combine the direct_check and
- # owner_check into a single EXISTS() clause, however it turns
+ # owner_check into a single IN (SELECT) clause, however it turns
# out query optimizer doesn't like it and forces a sequential
- # table scan. Constructing the query with separate EXISTS()
+ # table scan. Constructing the query with separate IN (SELECT)
# clauses enables it to use the index.
#
# see issue 13208 for details.
# Match a direct read permission link from the user to the record uuid
direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check})"
+ "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1)"
# Match a read permission for the user to the record's
# owner_uuid. This is so we can have a permissions table that
# other user owns.
owner_check = ""
if sql_table != "api_client_authorizations" and sql_table != "groups" then
- owner_check = "OR #{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
- "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check} AND traverse_owned) "
+ owner_check = "#{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
+ "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 AND traverse_owned) "
+
+ # We want to do owner_check before direct_check in the OR
+ # clause. The order of the OR clause isn't supposed to
+ # matter, but in practice, it does -- apparently in the
+ # absence of other hints, it uses the ordering from the query.
+ # For certain types of queries (like filtering on owner_uuid),
+ # every item will match the owner_check clause, so then
+ # Postgres will optimize out the direct_check entirely.
+ direct_check = " OR " + direct_check
end
links_cond = ""
"(#{sql_table}.head_uuid IN (#{user_uuids_subquery}) OR #{sql_table}.tail_uuid IN (#{user_uuids_subquery})))"
end
- sql_conds = "(#{direct_check} #{owner_check} #{links_cond}) #{exclude_trashed_records}"
+ sql_conds = "(#{owner_check} #{direct_check} #{links_cond}) #{trashed_check.empty? ? "" : "AND"} #{trashed_check}"
end
end
self.where(sql_conds,
- user_uuids: user_uuids,
+ user_uuids: all_user_uuids.collect{|c| c["target_uuid"]},
permission_link_classes: ['permission', 'resources'])
end
t.add :file_size_total
end
+ UNLOGGED_CHANGES = ['preserve_version', 'updated_at']
+
after_initialize do
@signatures_checked = false
@computed_pdh_for_manifest_text = false
snapshot = self.dup
snapshot.uuid = nil # Reset UUID so it's created as a new record
snapshot.created_at = self.created_at
+ snapshot.modified_at = self.modified_at_was
end
# Restore requested changes on the current version
changes.keys.each do |attr|
- if attr == 'preserve_version' && changes[attr].last == false
+ if attr == 'preserve_version' && changes[attr].last == false && !should_preserve_version
next # Ignore false assignment, once true it'll be true until next version
end
self.attributes = {attr => changes[attr].last}
if should_preserve_version
self.version += 1
- self.preserve_version = false
end
yield
if snapshot
snapshot.attributes = self.syncable_updates
leave_modified_by_user_alone do
- act_as_system_user do
- snapshot.save
+ leave_modified_at_alone do
+ act_as_system_user do
+ snapshot.save
+ end
end
end
end
end
end
+ def maybe_update_modified_by_fields
+ if !(self.changes.keys - ['updated_at', 'preserve_version']).empty?
+ super
+ end
+ end
+
def syncable_updates
updates = {}
if self.changes.any?
idle_threshold = Rails.configuration.Collections.PreserveVersionIfIdle
if !self.preserve_version_was &&
+ !self.preserve_version &&
(idle_threshold < 0 ||
(idle_threshold > 0 && self.modified_at_was > db_current_time-idle_threshold.seconds))
return false
self.current_version_uuid ||= self.uuid
true
end
+
+ def log_update
+ super unless (saved_changes.keys - UNLOGGED_CHANGES).empty?
+ end
end
def call_update_permissions
if self.link_class == 'permission'
update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
+ current_user.forget_cached_group_perms
end
end
def clear_permissions
if self.link_class == 'permission'
update_permissions tail_uuid, head_uuid, REVOKE_PERM, self.uuid
+ current_user.forget_cached_group_perms
end
end
before_update :verify_repositories_empty, :if => Proc.new {
username.nil? and username_changed?
}
- before_update :setup_on_activate
+ after_update :setup_on_activate
before_create :check_auto_admin
before_create :set_initial_username, :if => Proc.new {
MaterializedPermission.where("user_uuid = ? and target_uuid != ?", uuid, uuid).delete_all
end
+ def forget_cached_group_perms
+ @group_perms = nil
+ end
+
def remove_self_from_permissions
MaterializedPermission.where("target_uuid = ?", uuid).delete_all
check_permissions_against_full_refresh
# and perm_hash[:write] are true if this user can read and write
# objects owned by group_uuid.
def group_permissions(level=1)
- group_perms = {}
-
- user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$2"}
+ @group_perms ||= {}
+ if @group_perms.empty?
+ user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: 1}
- ActiveRecord::Base.connection.
- exec_query(%{
+ ActiveRecord::Base.connection.
+ exec_query(%{
SELECT target_uuid, perm_level
FROM #{PERMISSION_VIEW}
- WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= $2
+ WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= 1
},
- # "name" arg is a query label that appears in logs:
- "User.group_permissions",
- # "binds" arg is an array of [col_id, value] for '$1' vars:
- [[nil, uuid],
- [nil, level]]).
- rows.each do |group_uuid, max_p_val|
- group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
+ # "name" arg is a query label that appears in logs:
+ "User.group_permissions",
+ # "binds" arg is an array of [col_id, value] for '$1' vars:
+ [[nil, uuid]]).
+ rows.each do |group_uuid, max_p_val|
+ @group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
+ end
+ end
+
+ case level
+ when 1
+ @group_perms
+ when 2
+ @group_perms.select {|k,v| v[:write] }
+ when 3
+ @group_perms.select {|k,v| v[:manage] }
+ else
+ raise "level must be 1, 2 or 3"
end
- group_perms
end
# create links
- def setup(repo_name: nil, vm_uuid: nil)
- repo_perm = create_user_repo_link repo_name
- vm_login_perm = create_vm_login_permission_link(vm_uuid, username) if vm_uuid
+ def setup(repo_name: nil, vm_uuid: nil, send_notification_email: nil)
+ newly_invited = Link.where(tail_uuid: self.uuid,
+ head_uuid: all_users_group_uuid,
+ link_class: 'permission',
+ name: 'can_read').empty?
+
+ # Add can_read link from this user to "all users" which makes this
+ # user "invited"
group_perm = create_user_group_link
+ # Add git repo
+ repo_perm = if (!repo_name.nil? || Rails.configuration.Users.AutoSetupNewUsersWithRepository) and !username.nil?
+ repo_name ||= "#{username}/#{username}"
+ create_user_repo_link repo_name
+ end
+
+ # Add virtual machine
+ if vm_uuid.nil? and !Rails.configuration.Users.AutoSetupNewUsersWithVmUUID.empty?
+ vm_uuid = Rails.configuration.Users.AutoSetupNewUsersWithVmUUID
+ end
+
+ vm_login_perm = if vm_uuid && username
+ create_vm_login_permission_link(vm_uuid, username)
+ end
+
+ # Send welcome email
+ if send_notification_email.nil?
+ send_notification_email = Rails.configuration.Mail.SendUserSetupNotificationEmail
+ end
+
+ if newly_invited and send_notification_email and !Rails.configuration.Users.UserSetupMailText.empty?
+ begin
+ UserNotifier.account_is_setup(self).deliver_now
+ rescue => e
+ logger.warn "Failed to send email to #{self.email}: #{e}"
+ end
+ end
+
+ forget_cached_group_perms
+
return [repo_perm, vm_login_perm, group_perm, self].compact
end
self.prefs = {}
# mark the user as inactive
+ self.is_admin = false # can't be admin and inactive
self.is_active = false
+ forget_cached_group_perms
self.save!
end
# Automatically setup new user during creation
def auto_setup_new_user
setup
- if username
- create_vm_login_permission_link(Rails.configuration.Users.AutoSetupNewUsersWithVmUUID,
- username)
- repo_name = "#{username}/#{username}"
- if Rails.configuration.Users.AutoSetupNewUsersWithRepository and
- Repository.where(name: repo_name).first.nil?
- repo = Repository.create!(name: repo_name, owner_uuid: uuid)
- Link.create!(tail_uuid: uuid, head_uuid: repo.uuid,
- link_class: "permission", name: "can_manage")
- end
- end
end
# Send notification if the user saved profile for the first time
arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
-arvcfg.declare_config "API.RailsSessionSecretToken", NonemptyString, :secret_token
arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
arvcfg.declare_config "Users.AutoSetupNewUsersWithRepository", Boolean, :auto_setup_new_users_with_repository
# Rails.configuration.API["Blah"]
ConfigLoader.copy_into_config $arvados_config, config
ConfigLoader.copy_into_config $remaining_config, config
- secrets.secret_key_base = $arvados_config["API"]["RailsSessionSecretToken"]
+
+ # We don't rely on cookies for authentication, so instead of
+ # requiring a signing key in config, we assign a new random one at
+ # startup.
+ secrets.secret_key_base = rand(1<<255).to_s(36)
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+class RefreshTrashedGroups < ActiveRecord::Migration[5.2]
+ def change
+ # The original refresh_trashed query had a bug, it would insert
+ # all trashed rows, including those with null trash_at times.
+ # This went unnoticed because null trash_at behaved the same as
+ # not having those rows at all, but it is inefficient to fetch
+ # rows we'll never use. That bug is fixed in the original query
+ # but we need another migration to make sure it runs.
+ refresh_trashed
+ end
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+class RefreshPermissions < ActiveRecord::Migration[5.2]
+ def change
+ # There was a report of deadlocks resulting in failing permission
+ # updates. These failures should not have corrupted permissions
+ # (the failure should have rolled back the entire update) but we
+ # will refresh the permissions out of an abundance of caution.
+ refresh_permissions
+ end
+end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'fix_collection_versions_timestamps'
+
+class FixCollectionVersionsTimestamps < ActiveRecord::Migration[5.2]
+ def up
+ # Defined in a function for easy testing.
+ fix_collection_versions_timestamps
+ end
+
+ def down
+ # This migration is not reversible. However, the results are
+ # backwards compatible.
+ end
+end
SET xmloption = content;
SET client_min_messages = warning;
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: -
---
-
--- COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
--
-- Name: pg_trgm; Type: EXTENSION; Schema: -; Owner: -
--
('20190905151603'),
('20200501150153'),
('20200602141328'),
-('20200914203202');
+('20200914203202'),
+('20201103170213'),
+('20201105190435'),
+('20201202174753');
INSERT INTO #{TRASHED_GROUPS}
select ps.target_uuid as group_uuid, ps.trash_at from groups,
lateral project_subtree_with_trash_at(groups.uuid, groups.trash_at) ps
- where groups.owner_uuid like '_____-tpzed-_______________'
+ where groups.owner_uuid like '_____-tpzed-_______________' and ps.trash_at is not NULL
})
end
end
end
end
- api_client_auth.api_token
+ "v2/" + api_client_auth.uuid + "/" + api_client_auth.api_token
end
end
end
yield
ensure
Thread.current[:user] = user_was
+ if user_was
+ user_was.forget_cached_group_perms
+ end
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'set'
+
+include CurrentApiClient
+include ArvadosModelUpdates
+
+def fix_collection_versions_timestamps
+ act_as_system_user do
+ uuids = [].to_set
+ # Get UUIDs from collections with more than 1 version
+ Collection.where(version: 2).find_each(batch_size: 100) do |c|
+ uuids.add(c.current_version_uuid)
+ end
+ uuids.each do |uuid|
+ first_pair = true
+ # All versions of a particular collection get fixed together.
+ ActiveRecord::Base.transaction do
+ Collection.where(current_version_uuid: uuid).order(version: :desc).each_cons(2) do |c1, c2|
+ # Skip if the 2 newest versions' modified_at values are separate enough;
+ # this means that this collection doesn't require fixing, allowing for
+ # migration re-runs in case of transient problems.
+ break if first_pair && (c2.modified_at.to_f - c1.modified_at.to_f) > 1
+ first_pair = false
+ # Fix modified_at timestamps by assigning to N-1's value to N.
+ # Special case: the first version's modified_at will be == to created_at
+ leave_modified_by_user_alone do
+ leave_modified_at_alone do
+ c1.modified_at = c2.modified_at
+ c1.save!(validate: false)
+ if c2.version == 1
+ c2.modified_at = c2.created_at
+ c2.save!(validate: false)
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
\ No newline at end of file
ActiveRecord::Base.transaction do
- # "Conflicts with the ROW EXCLUSIVE, SHARE UPDATE EXCLUSIVE, SHARE
- # ROW EXCLUSIVE, EXCLUSIVE, and ACCESS EXCLUSIVE lock modes. This
- # mode protects a table against concurrent data changes."
- ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in SHARE MODE"
+ # "Conflicts with the ROW SHARE, ROW EXCLUSIVE, SHARE UPDATE
+ # EXCLUSIVE, SHARE, SHARE ROW EXCLUSIVE, EXCLUSIVE, and ACCESS
+ # EXCLUSIVE lock modes. This mode allows only concurrent ACCESS
+ # SHARE locks, i.e., only reads from the table can proceed in
+ # parallel with a transaction holding this lock mode."
+ ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in EXCLUSIVE MODE"
# Workaround for
# BUG #15160: planner overestimates number of rows in join when there are more than 200 rows coming from CTE
##### SSL - ward, 2012-10-15
require 'rubygems'
-require 'rails/commands/server'
+require 'rails/command'
require 'rack'
require 'webrick'
require 'webrick/https'
portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
- modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_client_uuid: zzzzz-ozdt8-teyxzyd8qllg11h
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2014-02-03T17:22:54Z
- updated_at: 2014-02-03T17:22:54Z
+ modified_at: 2014-02-03T18:22:54Z
+ updated_at: 2014-02-03T18:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
name: owned_by_active
version: 2
file_count: 1
file_size_total: 3
name: owned_by_active_with_file_stats
- version: 2
+ version: 1
collection_owned_by_active_past_version_1:
uuid: zzzzz-4zz18-znfnqtbbv4spast
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2014-02-03T15:22:54Z
- updated_at: 2014-02-03T15:22:54Z
+ modified_at: 2014-02-03T18:22:54Z
+ updated_at: 2014-02-03T18:22:54Z
manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
name: owned_by_active_version_1
version: 1
created_at: 2015-02-09T10:53:38Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2015-02-09T10:53:38Z
- updated_at: 2015-02-09T10:53:38Z
+ modified_at: 2015-02-09T10:55:38Z
+ updated_at: 2015-02-09T10:55:38Z
manifest_text: ". 4c6c2c0ac8aa0696edd7316a3be5ca3c+5 0:5:w\\040\\141\\040z\n"
name: "\"w a z\" file"
version: 2
created_at: 2015-02-09T10:53:38Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
- modified_at: 2015-02-09T10:53:38Z
- updated_at: 2015-02-09T10:53:38Z
+ modified_at: 2015-02-09T10:55:38Z
+ updated_at: 2015-02-09T10:55:38Z
manifest_text: ". 4d20280d5e516a0109768d49ab0f3318+3 0:3:waz\n"
name: "waz file"
version: 1
properties:
"http://schema.org/example": "value1"
+log_collection:
+ uuid: zzzzz-4zz18-logcollection01
+ current_version_uuid: zzzzz-4zz18-logcollection01
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: a real log collection for a completed container
+
+log_collection2:
+ uuid: zzzzz-4zz18-logcollection02
+ current_version_uuid: zzzzz-4zz18-logcollection02
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-10-29T00:51:44.075594000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-10-29T00:51:44.072109000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: another real log collection for a completed container
+
+diagnostics_request_container_log_collection:
+ uuid: zzzzz-4zz18-diagcompreqlog1
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog1
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:44.007557000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:44.005381000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0001
+
+hasher1_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash001
+ current_version_uuid: zzzzz-4zz18-dlogcollhash001
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:16:55.272606000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:16:55.267006000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher1 log collection
+
+hasher2_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash002
+ current_version_uuid: zzzzz-4zz18-dlogcollhash002
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:23.547251000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:23.545275000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher2 log collection
+
+hasher3_log_collection:
+ uuid: zzzzz-4zz18-dlogcollhash003
+ current_version_uuid: zzzzz-4zz18-dlogcollhash003
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-02T00:20:38.789204000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-02T00:20:38.787329000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: hasher3 log collection
+
+diagnostics_request_container_log_collection2:
+ uuid: zzzzz-4zz18-diagcompreqlog2
+ current_version_uuid: zzzzz-4zz18-diagcompreqlog2
+ portable_data_hash: 680c855fd6cf2c78778b3728b268925a+475
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2020-11-03T16:17:53.351593000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2020-11-03T16:17:53.346969000Z
+ manifest_text: ". 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n./log\\040for\\040container\\040ce8i5-dz642-h4kd64itncdcz8l 8c12f5f5297b7337598170c6f531fcee+7882 0:0:arv-mount.txt 0:1910:container.json 1910:1264:crunch-run.txt 3174:1005:crunchstat.txt 4179:659:hoststat.txt 4838:2811:node-info.txt 7649:233:node.json 0:0:stderr.txt\n"
+ name: Container log for request zzzzz-xvhdp-diagnostics0002
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
output_path: test
command: ["arvados-cwl-runner", "echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainr2
+ log_uuid: zzzzz-4zz18-logcollection02
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
ram: 123
+completed_diagnostics:
+ name: CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-diagcompreqlog1
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag1hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag1hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag1hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0001
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics2:
+ name: Copy of CWL diagnostics hasher
+ uuid: zzzzz-xvhdp-diagnostics0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 1
+ created_at: 2020-11-03T15:54:30.098485000Z
+ modified_at: 2020-11-03T16:17:53.406809000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-diagcompreqlog2
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 1342177280
+ API: true
+
+completed_diagnostics_hasher1_reuse:
+ name: hasher1
+ uuid: zzzzz-xvhdp-diag2hasher0001
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:03:50.229364000Z
+ modified_at: 2020-11-02T00:20:44.041122000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher1
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher1
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash001
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher2_reuse:
+ name: hasher2
+ uuid: zzzzz-xvhdp-diag2hasher0002
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:17:07.067464000Z
+ modified_at: 2020-11-02T00:20:23.557498000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher2
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher2
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash002
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 2
+ ram: 2684354560
+ API: true
+
+completed_diagnostics_hasher3_reuse:
+ name: hasher3
+ uuid: zzzzz-xvhdp-diag2hasher0003
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ state: Final
+ priority: 500
+ created_at: 2020-11-02T00:20:30.960251000Z
+ modified_at: 2020-11-02T00:20:38.799377000Z
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ output_name: Output for step hasher3
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ container_uuid: zzzzz-dz642-diagcomphasher3
+ requesting_container_uuid: zzzzz-dz642-diagcompreq0002
+ log_uuid: zzzzz-4zz18-dlogcollhash003
+ output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
+ runtime_constraints:
+ vcpus: 1
+ ram: 2684354560
+ API: true
+
requester:
uuid: zzzzz-xvhdp-9zacv3o1xw6sxz5
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
vcpus: 1
ram: 123
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
mounts:
/var/lib/cwl/cwl.input.json:
output_path: test
command: ["echo", "hello"]
container_uuid: zzzzz-dz642-compltcontainer
- log_uuid: zzzzz-4zz18-y9vne9npefyxh8g
+ log_uuid: zzzzz-4zz18-logcollection01
output_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
runtime_constraints:
vcpus: 1
secret_mounts: {}
secret_mounts_md5: 99914b932bd37a50b983c5e7c90ae93b
+diagnostics_completed_requester:
+ uuid: zzzzz-dz642-diagcompreq0001
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:03:50.192697000Z
+ modified_at: 2020-11-02T00:20:43.987275000Z
+ started_at: 2020-11-02T00:08:07.186711000Z
+ finished_at: 2020-11-02T00:20:43.975416000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 6129e376cb05c942f75a0c36083383e8+244
+ output: 1f4b0bc7583c2a7f9102c395f4ffc5e3+45
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
+diagnostics_completed_hasher1:
+ uuid: zzzzz-dz642-diagcomphasher1
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:08:18.829222000Z
+ modified_at: 2020-11-02T00:16:55.142023000Z
+ started_at: 2020-11-02T00:16:52.375871000Z
+ finished_at: 2020-11-02T00:16:55.105985000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: fed8fb19fe8e3a320c29fed0edab12dd+220
+ output: d3a687732e84061f3bae15dc7e313483+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/9f26a86b6030a69ad222cf67d71c9502+65/hasher-input-file.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_hasher2:
+ uuid: zzzzz-dz642-diagcomphasher2
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:17:07.026493000Z
+ modified_at: 2020-11-02T00:20:23.505908000Z
+ started_at: 2020-11-02T00:20:21.513185000Z
+ finished_at: 2020-11-02T00:20:23.478317000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 4fc03b95fc2646b0dec7383dbb7d56d8+221
+ output: 6bd770f6cf8f83e7647c602eecfaeeb8+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/d3a687732e84061f3bae15dc7e313483+62/hasher1.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 2
+
+diagnostics_completed_hasher3:
+ uuid: zzzzz-dz642-diagcomphasher3
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 562948349145881771
+ created_at: 2020-11-02T00:20:30.943856000Z
+ modified_at: 2020-11-02T00:20:38.746541000Z
+ started_at: 2020-11-02T00:20:36.748957000Z
+ finished_at: 2020-11-02T00:20:38.732199000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: 1eeaf70de0f65b1346e54c59f09e848d+210
+ output: 11b5fdaa380102e760c3eb6de80a9876+62
+ output_path: /var/spool/cwl
+ command: [
+ "md5sum",
+ "/keep/6bd770f6cf8f83e7647c602eecfaeeb8+62/hasher2.md5sum.txt"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 268435456
+ vcpus: 1
+
+diagnostics_completed_requester2:
+ uuid: zzzzz-dz642-diagcompreq0002
+ owner_uuid: zzzzz-tpzed-000000000000000
+ state: Complete
+ exit_code: 0
+ priority: 1124295487972526
+ created_at: 2020-11-03T15:54:36.504661000Z
+ modified_at: 2020-11-03T16:17:53.242868000Z
+ started_at: 2020-11-03T16:09:51.123659000Z
+ finished_at: 2020-11-03T16:17:53.220358000Z
+ container_image: d967ef4a1ca90a096a39f5ce68e4a2e7+261
+ cwd: /var/spool/cwl
+ log: f1933bf5191f576613ea7f65bd0ead53+244
+ output: 941b71a57208741ce8742eca62352fb1+123
+ output_path: /var/spool/cwl
+ command: [
+ "arvados-cwl-runner",
+ "--local",
+ "--api=containers",
+ "--no-log-timestamps",
+ "--disable-validate",
+ "--disable-color",
+ "--eval-timeout=20",
+ "--thread-count=1",
+ "--disable-reuse",
+ "--collection-cache-size=256",
+ "--on-error=continue",
+ "/var/lib/cwl/workflow.json#main",
+ "/var/lib/cwl/cwl.input.json"
+ ]
+ runtime_constraints:
+ API: true
+ keep_cache_ram: 268435456
+ ram: 1342177280
+ vcpus: 1
+
requester:
uuid: zzzzz-dz642-requestingcntnr
owner_uuid: zzzzz-tpzed-000000000000000
end
[:admin, :active].each do |user|
- test "get trashed collection via filters and #{user} user" do
+ test "get trashed collection via filters and #{user} user without including its past versions" do
uuid = 'zzzzz-4zz18-mto52zx1s7sn3ih' # expired_collection
authorize_with user
get :index, params: {
refute_includes found_uuids, specimens(:in_asubproject).uuid, "specimen appeared unexpectedly in home project"
end
+ test "list collections in home project" do
+ authorize_with :active
+ get(:contents, params: {
+ format: :json,
+ filters: [
+ ['uuid', 'is_a', 'arvados#collection'],
+ ],
+ limit: 200,
+ id: users(:active).uuid,
+ })
+ assert_response :success
+ found_uuids = json_response['items'].collect { |i| i['uuid'] }
+ assert_includes found_uuids, collections(:collection_owned_by_active).uuid, "collection did not appear in home project"
+ refute_includes found_uuids, collections(:collection_owned_by_active_past_version_1).uuid, "collection appeared unexpectedly in home project"
+ end
+
+ test "list collections in home project, including old versions" do
+ authorize_with :active
+ get(:contents, params: {
+ format: :json,
+ include_old_versions: true,
+ filters: [
+ ['uuid', 'is_a', 'arvados#collection'],
+ ],
+ limit: 200,
+ id: users(:active).uuid,
+ })
+ assert_response :success
+ found_uuids = json_response['items'].collect { |i| i['uuid'] }
+ assert_includes found_uuids, collections(:collection_owned_by_active).uuid, "collection did not appear in home project"
+ assert_includes found_uuids, collections(:collection_owned_by_active_past_version_1).uuid, "old collection version did not appear in home project"
+ end
+
test "user with project read permission can see project collections" do
authorize_with :project_viewer
get :contents, params: {
end
end
- test "Collection contents don't include manifest_text" do
+ test "Collection contents don't include manifest_text or unsigned_manifest_text" do
authorize_with :active
get :contents, params: {
id: groups(:aproject).uuid,
refute(json_response["items"].any? { |c| not c["portable_data_hash"] },
"response included an item without a portable data hash")
refute(json_response["items"].any? { |c| c.include?("manifest_text") },
- "response included an item with a manifest text")
+ "response included an item with manifest_text")
+ refute(json_response["items"].any? { |c| c.include?("unsigned_manifest_text") },
+ "response included an item with unsigned_manifest_text")
end
test 'get writable_by list for owned group' do
group_index_params = discovery_doc['resources']['groups']['methods']['index']['parameters']
group_contents_params = discovery_doc['resources']['groups']['methods']['contents']['parameters']
- assert_equal group_contents_params.keys.sort, (group_index_params.keys - ['select'] + ['uuid', 'recursive', 'include']).sort
+ assert_equal group_contents_params.keys.sort, (group_index_params.keys - ['select'] + ['uuid', 'recursive', 'include', 'include_old_versions']).sort
recursive_param = group_contents_params['recursive']
assert_equal 'boolean', recursive_param['type']
assert_equal Hash, json_response['properties'].class, 'Collection properties attribute should be of type hash'
assert_equal 'value_1', json_response['properties']['property_1']
end
+
+ test "update collection with versioning enabled and using preserve_version" do
+ Rails.configuration.Collections.CollectionVersioning = true
+ Rails.configuration.Collections.PreserveVersionIfIdle = -1 # Disable auto versioning
+
+ signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
+ post "/arvados/v1/collections",
+ params: {
+ format: :json,
+ collection: {
+ name: 'Test collection',
+ manifest_text: signed_manifest,
+ }.to_json,
+ },
+ headers: auth(:active)
+ assert_response 200
+ assert_not_nil json_response['uuid']
+ assert_equal 1, json_response['version']
+ assert_equal false, json_response['preserve_version']
+
+ # Versionable update including preserve_version=true should create a new
+ # version that will also be persisted.
+ put "/arvados/v1/collections/#{json_response['uuid']}",
+ params: {
+ format: :json,
+ collection: {
+ name: 'Test collection v2',
+ preserve_version: true,
+ }.to_json,
+ },
+ headers: auth(:active)
+ assert_response 200
+ assert_equal 2, json_response['version']
+ assert_equal true, json_response['preserve_version']
+
+ # 2nd versionable update including preserve_version=true should create a new
+ # version that will also be persisted.
+ put "/arvados/v1/collections/#{json_response['uuid']}",
+ params: {
+ format: :json,
+ collection: {
+ name: 'Test collection v3',
+ preserve_version: true,
+ }.to_json,
+ },
+ headers: auth(:active)
+ assert_response 200
+ assert_equal 3, json_response['version']
+ assert_equal true, json_response['preserve_version']
+
+ # 3rd versionable update without including preserve_version should create a new
+ # version that will have its preserve_version attr reset to false.
+ put "/arvados/v1/collections/#{json_response['uuid']}",
+ params: {
+ format: :json,
+ collection: {
+ name: 'Test collection v4',
+ }.to_json,
+ },
+ headers: auth(:active)
+ assert_response 200
+ assert_equal 4, json_response['version']
+ assert_equal false, json_response['preserve_version']
+
+ # 4th versionable update without including preserve_version=true should NOT
+ # create a new version.
+ put "/arvados/v1/collections/#{json_response['uuid']}",
+ params: {
+ format: :json,
+ collection: {
+ name: 'Test collection v5?',
+ }.to_json,
+ },
+ headers: auth(:active)
+ assert_response 200
+ assert_equal 4, json_response['version']
+ assert_equal false, json_response['preserve_version']
+ end
end
assert_equal 'barney', json_response['username']
end
- test 'get inactive user from Login cluster when AutoSetupNewUsers is set' do
- Rails.configuration.Login.LoginCluster = 'zbbbb'
- Rails.configuration.Users.AutoSetupNewUsers = true
- @stub_content = {
- uuid: 'zbbbb-tpzed-000000000000001',
- email: 'foo@example.com',
- username: 'barney',
- is_admin: false,
- is_active: false,
- is_invited: false,
- }
- get '/arvados/v1/users/current',
- params: {format: 'json'},
- headers: auth(remote: 'zbbbb')
- assert_response :success
- assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid']
- assert_equal false, json_response['is_admin']
- assert_equal false, json_response['is_active']
- assert_equal false, json_response['is_invited']
- assert_equal 'foo@example.com', json_response['email']
- assert_equal 'barney', json_response['username']
+ [true, false].each do |trusted|
+ [true, false].each do |logincluster|
+ [true, false].each do |admin|
+ [true, false].each do |active|
+ [true, false].each do |autosetup|
+ [true, false].each do |invited|
+ test "get invited=#{invited}, active=#{active}, admin=#{admin} user from #{if logincluster then "Login" else "peer" end} cluster when AutoSetupNewUsers=#{autosetup} ActivateUsers=#{trusted}" do
+ Rails.configuration.Login.LoginCluster = 'zbbbb' if logincluster
+ Rails.configuration.RemoteClusters['zbbbb'].ActivateUsers = trusted
+ Rails.configuration.Users.AutoSetupNewUsers = autosetup
+ @stub_content = {
+ uuid: 'zbbbb-tpzed-000000000000001',
+ email: 'foo@example.com',
+ username: 'barney',
+ is_admin: admin,
+ is_active: active,
+ is_invited: invited,
+ }
+ get '/arvados/v1/users/current',
+ params: {format: 'json'},
+ headers: auth(remote: 'zbbbb')
+ assert_response :success
+ assert_equal 'zbbbb-tpzed-000000000000001', json_response['uuid']
+ assert_equal (logincluster && admin && invited && active), json_response['is_admin']
+ assert_equal (invited and (logincluster || trusted || autosetup)), json_response['is_invited']
+ assert_equal (invited and (logincluster || trusted) and active), json_response['is_active']
+ assert_equal 'foo@example.com', json_response['email']
+ assert_equal 'barney', json_response['username']
+ end
+ end
+ end
+ end
+ end
+ end
end
- test 'get active user from Login cluster when AutoSetupNewUsers is set' do
+ test 'get active user from Login cluster when AutoSetupNewUsers is set' do
Rails.configuration.Login.LoginCluster = 'zbbbb'
Rails.configuration.Users.AutoSetupNewUsers = true
@stub_content = {
def set_user_from_auth(auth_name)
client_auth = api_client_authorizations(auth_name)
+ client_auth.user.forget_cached_group_perms
Thread.current[:api_client_authorization] = client_auth
Thread.current[:api_client] = client_auth.api_client
Thread.current[:user] = client_auth.user
require 'test_helper'
require 'sweep_trashed_objects'
+require 'fix_collection_versions_timestamps'
class CollectionTest < ActiveSupport::TestCase
include DbCurrentTime
end
end
- test "preserve_version=false assignment is ignored while being true and not producing a new version" do
+ test "preserve_version updates" do
Rails.configuration.Collections.CollectionVersioning = true
- Rails.configuration.Collections.PreserveVersionIfIdle = 3600
+ Rails.configuration.Collections.PreserveVersionIfIdle = -1 # disabled
act_as_user users(:active) do
# Set up initial collection
c = create_collection 'foo', Encoding::US_ASCII
assert_equal false, c.preserve_version
# This update shouldn't produce a new version, as the idle time is not up
c.update_attributes!({
- 'name' => 'bar',
- 'preserve_version' => true
+ 'name' => 'bar'
})
c.reload
assert_equal 1, c.version
assert_equal 'bar', c.name
+ assert_equal false, c.preserve_version
+ # This update should produce a new version, even if the idle time is not up
+ # and also keep the preserve_version=true flag to persist it.
+ c.update_attributes!({
+ 'name' => 'baz',
+ 'preserve_version' => true
+ })
+ c.reload
+ assert_equal 2, c.version
+ assert_equal 'baz', c.name
assert_equal true, c.preserve_version
# Make sure preserve_version is not disabled after being enabled, unless
# a new version is created.
+ # This is a non-versionable update
c.update_attributes!({
'preserve_version' => false,
'replication_desired' => 2
})
c.reload
- assert_equal 1, c.version
+ assert_equal 2, c.version
assert_equal 2, c.replication_desired
assert_equal true, c.preserve_version
- c.update_attributes!({'name' => 'foobar'})
+ # This is a versionable update
+ c.update_attributes!({
+ 'preserve_version' => false,
+ 'name' => 'foobar'
+ })
c.reload
- assert_equal 2, c.version
+ assert_equal 3, c.version
assert_equal false, c.preserve_version
assert_equal 'foobar', c.name
+ # Flipping only 'preserve_version' to true doesn't create a new version
+ c.update_attributes!({'preserve_version' => true})
+ c.reload
+ assert_equal 3, c.version
+ assert_equal true, c.preserve_version
+ end
+ end
+
+ test "preserve_version updates don't change modified_at timestamp" do
+ act_as_user users(:active) do
+ c = create_collection 'foo', Encoding::US_ASCII
+ assert c.valid?
+ assert_equal false, c.preserve_version
+ modified_at = c.modified_at.to_f
+ c.update_attributes!({'preserve_version' => true})
+ c.reload
+ assert_equal true, c.preserve_version
+ assert_equal modified_at, c.modified_at.to_f,
+ 'preserve_version updates should not trigger modified_at changes'
end
end
# Set up initial collection
c = create_collection 'foo', Encoding::US_ASCII
assert c.valid?
+ original_version_modified_at = c.modified_at.to_f
# Make changes so that a new version is created
c.update_attributes!({'name' => 'bar'})
c.reload
version_creation_datetime = c_old.modified_at.to_f
assert_equal c.created_at.to_f, c_old.created_at.to_f
- # Current version is updated just a few milliseconds before the version is
- # saved on the database.
- assert_operator c.modified_at.to_f, :<, version_creation_datetime
+ assert_equal original_version_modified_at, version_creation_datetime
# Make update on current version so old version get the attribute synced;
# its modified_at should not change.
end
end
+ # Bug #17152 - This test relies on fixtures simulating the problem.
+ test "migration fixing collection versions' modified_at timestamps" do
+ versioned_collection_fixtures = [
+ collections(:w_a_z_file).uuid,
+ collections(:collection_owned_by_active).uuid
+ ]
+ versioned_collection_fixtures.each do |uuid|
+ cols = Collection.where(current_version_uuid: uuid).order(version: :desc)
+ assert_equal cols.size, 2
+ # cols[0] -> head version // cols[1] -> old version
+ assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :==, 0
+ assert cols[1].modified_at != cols[1].created_at
+ end
+ fix_collection_versions_timestamps
+ versioned_collection_fixtures.each do |uuid|
+ cols = Collection.where(current_version_uuid: uuid).order(version: :desc)
+ assert_equal cols.size, 2
+ # cols[0] -> head version // cols[1] -> old version
+ assert_operator (cols[0].modified_at.to_f - cols[1].modified_at.to_f), :>, 1
+ assert_operator cols[1].modified_at, :==, cols[1].created_at
+ end
+ end
+
test "past versions should not be directly updatable" do
Rails.configuration.Collections.CollectionVersioning = true
Rails.configuration.Collections.PreserveVersionIfIdle = 0
class CreateSuperUserTokenTest < ActiveSupport::TestCase
include CreateSuperUserToken
- test "create superuser token twice and expect same resutls" do
+ test "create superuser token twice and expect same results" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/atesttoken$/, token1)
# Create token again; this time, we should get the one created earlier
token2 = create_superuser_token
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
# Create token again with some other string and expect the existing superuser token back
token2 = create_superuser_token 'someothertokenstring'
assert_equal token1, token2
end
- test "create superuser token twice and expect same results" do
- # Create a token with some string
- token1 = create_superuser_token 'atesttoken'
- assert_not_nil token1
- assert_equal token1, 'atesttoken'
-
- # Create token again with that same superuser token and expect it back
- token2 = create_superuser_token 'atesttoken'
- assert_not_nil token2
- assert_equal token1, token2
- end
-
test "create superuser token and invoke again with some other valid token" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
su_token = api_client_authorizations("system_user").api_token
token2 = create_superuser_token su_token
- assert_equal token2, su_token
+ assert_equal token2.split('/')[2], su_token
end
test "create superuser token, expire it, and create again" do
# Create a token with some string
token1 = create_superuser_token 'atesttoken'
assert_not_nil token1
- assert_equal token1, 'atesttoken'
+ assert_match(/\/atesttoken$/, token1)
# Expire this token and call create again; expect a new token created
- apiClientAuth = ApiClientAuthorization.where(api_token: token1).first
+ apiClientAuth = ApiClientAuthorization.where(api_token: 'atesttoken').first
+ refute_nil apiClientAuth
Thread.current[:user] = users(:admin)
apiClientAuth.update_attributes expires_at: '2000-10-10'
assert_logged(auth, :update)
end
+ test "don't log changes only to Collection.preserve_version" do
+ set_user_from_auth :admin_trustedclient
+ col = collections(:collection_owned_by_active)
+ start_log_count = get_logs_about(col).size
+ assert_equal false, col.preserve_version
+ col.preserve_version = true
+ col.save!
+ assert_equal(start_log_count, get_logs_about(col).size,
+ "log count changed after updating Collection.preserve_version")
+ col.name = 'updated by admin'
+ col.save!
+ assert_logged(col, :update)
+ end
+
test "token isn't included in ApiClientAuthorization logs" do
set_user_from_auth :admin_trustedclient
auth = ApiClientAuthorization.new
[false, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", true, true, "ad9"],
[false, active_notify_list, inactive_notify_list, "&4a_d9.@example.com", false, false, "ad9"],
].each do |active, new_user_recipients, inactive_recipients, email, auto_setup_vm, auto_setup_repo, expect_username|
- test "create new user with auto setup #{active} #{email} #{auto_setup_vm} #{auto_setup_repo}" do
+ test "create new user with auto setup active=#{active} email=#{email} vm=#{auto_setup_vm} repo=#{auto_setup_repo}" do
set_user_from_auth :admin
Rails.configuration.Users.AutoSetupNewUsers = true
Rails.configuration.Users.AutoSetupNewUsersWithRepository),
named_repo.uuid, user.uuid, "permission", "can_manage")
end
+
# Check for VM login.
if (auto_vm_uuid = Rails.configuration.Users.AutoSetupNewUsersWithVmUUID) != ""
verify_link_exists(can_setup, auto_vm_uuid, user.uuid,
tail_uuid: tail_uuid,
link_class: link_class,
name: link_name)
- assert_equal link_exists, all_links.any?, "Link #{'not' if link_exists} found for #{link_name} #{link_class} #{property_value}"
+ assert_equal link_exists, all_links.any?, "Link#{' not' if link_exists} found for #{link_name} #{link_class} #{property_value}"
if link_exists && property_name && property_value
all_links.each do |link|
assert_equal true, all_links.first.properties[property_name].start_with?(property_value), 'Property not found in link'
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+[Unit]
+Description=Arvados Crunch Dispatcher for LOCAL service
+Documentation=https://doc.arvados.org/
+After=network.target
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
+StartLimitInterval=0
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+EnvironmentFile=-/etc/arvados/crunch-dispatch-local-credentials
+ExecStart=/usr/bin/crunch-dispatch-local -poll-interval=1 -crunch-run-command=/usr/bin/crunch-run
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
+Restart=always
+RestartSec=1
+LimitNOFILE=1000000
+
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+fpm_depends+=(crunch-run)
// Cancelled or Complete. See https://dev.arvados.org/issues/10979
func (disp *Dispatcher) checkSqueueForOrphans() {
for _, uuid := range disp.sqCheck.All() {
- if !containerUuidPattern.MatchString(uuid) {
+ if !containerUuidPattern.MatchString(uuid) || !strings.HasPrefix(uuid, disp.cluster.ClusterID) {
continue
}
err := disp.TrackContainer(uuid)
import time
import os
import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ return getver
def git_version_at_commit():
- curdir = os.path.dirname(os.path.abspath(__file__))
+ curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
def get_version(setup_dir, module):
env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
# SPDX-License-Identifier: Apache-2.0
case "$TARGET" in
- debian9 | ubuntu1604)
+ ubuntu1604)
fpm_depends+=()
;;
debian* | ubuntu*)
+++ /dev/null
-../../sdk/python/gittaggers.py
\ No newline at end of file
1. Add this Arvados repository to your sources list::
- deb http://apt.arvados.org/ jessie main
+ deb http://apt.arvados.org/buster buster main
2. Update your package list.
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../../sdk/python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
def get_version(setup_dir, module):
env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+++ /dev/null
-../../sdk/python/gittaggers.py
\ No newline at end of file
func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvados.CollectionFileSystem) error {
c.setupOnce.Do(c.setup)
- if m, err := fs.MarshalManifest("."); err != nil || m == coll.ManifestText {
+ m, err := fs.MarshalManifest(".")
+ if err != nil || m == coll.ManifestText {
return err
- } else {
- coll.ManifestText = m
}
+ coll.ManifestText = m
var updated arvados.Collection
defer c.pdhs.Remove(coll.UUID)
- err := client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{
+ err = client.RequestAndDecode(&updated, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{
"collection": map[string]string{
"manifest_text": coll.ManifestText,
},
//
// Download URLs
//
-// The following "same origin" URL patterns are supported for public
-// collections and collections shared anonymously via secret links
-// (i.e., collections which can be served by keep-web without making
-// use of any implicit credentials like cookies). See "Same-origin
-// URLs" below.
-//
-// http://collections.example.com/c=uuid_or_pdh/path/file.txt
-// http://collections.example.com/c=uuid_or_pdh/t=TOKEN/path/file.txt
-//
-// The following "multiple origin" URL patterns are supported for all
-// collections:
-//
-// http://uuid_or_pdh--collections.example.com/path/file.txt
-// http://uuid_or_pdh--collections.example.com/t=TOKEN/path/file.txt
-//
-// In the "multiple origin" form, the string "--" can be replaced with
-// "." with identical results (assuming the downstream proxy is
-// configured accordingly). These two are equivalent:
-//
-// http://uuid_or_pdh--collections.example.com/path/file.txt
-// http://uuid_or_pdh.collections.example.com/path/file.txt
-//
-// The first form (with "--" instead of ".") avoids the cost and
-// effort of deploying a wildcard TLS certificate for
-// *.collections.example.com at sites that already have a wildcard
-// certificate for *.example.com. The second form is likely to be
-// easier to configure, and more efficient to run, on a downstream
-// proxy.
-//
-// In all of the above forms, the "collections.example.com" part can
-// be anything at all: keep-web itself ignores everything after the
-// first "." or "--". (Of course, in order for clients to connect at
-// all, DNS and any relevant proxies must be configured accordingly.)
-//
-// In all of the above forms, the "uuid_or_pdh" part can be either a
-// collection UUID or a portable data hash with the "+" character
-// optionally replaced by "-". (When "uuid_or_pdh" appears in the
-// domain name, replacing "+" with "-" is mandatory, because "+" is
-// not a valid character in a domain name.)
-//
-// In all of the above forms, a top level directory called "_" is
-// skipped. In cases where the "path/file.txt" part might start with
-// "t=" or "c=" or "_/", links should be constructed with a leading
-// "_/" to ensure the top level directory is not interpreted as a
-// token or collection ID.
-//
-// Assuming there is a collection with UUID
-// zzzzz-4zz18-znfnqtbbv4spc3w and portable data hash
-// 1f4b0bc7583c2a7f9102c395f4ffc5e3+45, the following URLs are
-// interchangeable:
-//
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/foo/bar.txt
-// http://zzzzz-4zz18-znfnqtbbv4spc3w.collections.example.com/_/foo/bar.txt
-// http://zzzzz-4zz18-znfnqtbbv4spc3w--collections.example.com/_/foo/bar.txt
-//
-// The following URLs are read-only, but otherwise interchangeable
-// with the above:
-//
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--foo.example.com/foo/bar.txt
-// http://1f4b0bc7583c2a7f9102c395f4ffc5e3-45--.invalid/foo/bar.txt
-// http://collections.example.com/by_id/1f4b0bc7583c2a7f9102c395f4ffc5e3%2B45/foo/bar.txt
-// http://collections.example.com/by_id/zzzzz-4zz18-znfnqtbbv4spc3w/foo/bar.txt
-//
-// If the collection is named "MyCollection" and located in a project
-// called "MyProject" which is in the home project of a user with
-// username is "bob", the following read-only URL is also available
-// when authenticating as bob:
-//
-// http://collections.example.com/users/bob/MyProject/MyCollection/foo/bar.txt
-//
-// An additional form is supported specifically to make it more
-// convenient to maintain support for existing Workbench download
-// links:
-//
-// http://collections.example.com/collections/download/uuid_or_pdh/TOKEN/foo/bar.txt
-//
-// A regular Workbench "download" link is also accepted, but
-// credentials passed via cookie, header, etc. are ignored. Only
-// public data can be served this way:
-//
-// http://collections.example.com/collections/uuid_or_pdh/foo/bar.txt
-//
-// Collections can also be accessed (read-only) via "/by_id/X" where X
-// is a UUID or portable data hash.
-//
-// Authorization mechanisms
-//
-// A token can be provided in an Authorization header:
-//
-// Authorization: OAuth2 o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
-//
-// A base64-encoded token can be provided in a cookie named "api_token":
-//
-// Cookie: api_token=bzA3ajRweDdSbEpLNEN1TVlwN0MwTERUNEN6UjFKMXFCRTVBdm83ZUNjVWpPVGlreEs=
-//
-// A token can be provided in an URL-encoded query string:
-//
-// GET /foo/bar.txt?api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
-//
-// A suitably encoded token can be provided in a POST body if the
-// request has a content type of application/x-www-form-urlencoded or
-// multipart/form-data:
-//
-// POST /foo/bar.txt
-// Content-Type: application/x-www-form-urlencoded
-// [...]
-// api_token=o07j4px7RlJK4CuMYp7C0LDT4CzR1J1qBE5Avo7eCcUjOTikxK
-//
-// If a token is provided in a query string or in a POST request, the
-// response is an HTTP 303 redirect to an equivalent GET request, with
-// the token stripped from the query string and added to a cookie
-// instead.
-//
-// Indexes
-//
-// Keep-web returns a generic HTML index listing when a directory is
-// requested with the GET method. It does not serve a default file
-// like "index.html". Directory listings are also returned for WebDAV
-// PROPFIND requests.
-//
-// Compatibility
-//
-// Client-provided authorization tokens are ignored if the client does
-// not provide a Host header.
-//
-// In order to use the query string or a POST form authorization
-// mechanisms, the client must follow 303 redirects; the client must
-// accept cookies with a 303 response and send those cookies when
-// performing the redirect; and either the client or an intervening
-// proxy must resolve a relative URL ("//host/path") if given in a
-// response Location header.
-//
-// Intranet mode
-//
-// Normally, Keep-web accepts requests for multiple collections using
-// the same host name, provided the client's credentials are not being
-// used. This provides insufficient XSS protection in an installation
-// where the "anonymously accessible" data is not truly public, but
-// merely protected by network topology.
-//
-// In such cases -- for example, a site which is not reachable from
-// the internet, where some data is world-readable from Arvados's
-// perspective but is intended to be available only to users within
-// the local network -- the downstream proxy should configured to
-// return 401 for all paths beginning with "/c=".
-//
-// Same-origin URLs
-//
-// Without the same-origin protection outlined above, a web page
-// stored in collection X could execute JavaScript code that uses the
-// current viewer's credentials to download additional data from
-// collection Y -- data which is accessible to the current viewer, but
-// not to the author of collection X -- from the same origin
-// (``https://collections.example.com/'') and upload it to some other
-// site chosen by the author of collection X.
+// See http://doc.arvados.org/api/keep-web-urls.html
//
// Attachment-Only host
//
var urlPDHDecoder = strings.NewReplacer(" ", "+", "-", "+")
+var notFoundMessage = "404 Not found\r\n\r\nThe requested path was not found, or you do not have permission to access it.\r"
+var unauthorizedMessage = "401 Unauthorized\r\n\r\nA valid Arvados token must be provided to access this resource.\r"
+
// parseCollectionIDFromURL returns a UUID or PDH if s is a UUID or a
// PDH (even if it is a PDH with "+" replaced by " " or "-");
// otherwise "".
}
if collectionID == "" && !useSiteFS {
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
}
formToken := r.FormValue("api_token")
- if formToken != "" && r.Header.Get("Origin") != "" && attachment && r.URL.Query().Get("api_token") == "" {
- // The client provided an explicit token in the POST
- // body. The Origin header indicates this *might* be
- // an AJAX request, in which case redirect-with-cookie
- // won't work: we should just serve the content in the
- // POST response. This is safe because:
- //
- // * We're supplying an attachment, not inline
- // content, so we don't need to convert the POST to
- // a GET and avoid the "really resubmit form?"
- // problem.
+ origin := r.Header.Get("Origin")
+ cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
+ safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
+ safeAttachment := attachment && r.URL.Query().Get("api_token") == ""
+ if formToken == "" {
+ // No token to use or redact.
+ } else if safeAjax || safeAttachment {
+ // If this is a cross-origin request, the URL won't
+ // appear in the browser's address bar, so
+ // substituting a clipboard-safe URL is pointless.
+ // Redirect-with-cookie wouldn't work anyway, because
+ // it's not safe to allow third-party use of our
+ // cookie.
//
- // * The token isn't embedded in the URL, so we don't
- // need to worry about bookmarks and copy/paste.
+ // If we're supplying an attachment, we don't need to
+ // convert POST to GET to avoid the "really resubmit
+ // form?" problem, so provided the token isn't
+ // embedded in the URL, there's no reason to do
+ // redirect-with-cookie in this case either.
reqTokens = append(reqTokens, formToken)
- } else if formToken != "" && browserMethod[r.Method] {
- // The client provided an explicit token in the query
- // string, or a form in POST body. We must put the
- // token in an HttpOnly cookie, and redirect to the
- // same URL with the query param redacted and method =
- // GET.
+ } else if browserMethod[r.Method] {
+ // If this is a page view, and the client provided a
+ // token via query string or POST body, we must put
+ // the token in an HttpOnly cookie, and redirect to an
+ // equivalent URL with the query param redacted and
+ // method = GET.
h.seeOtherWithCookie(w, r, "", credentialsOK)
return
}
// for additional credentials would just be
// confusing), or we don't even accept
// credentials at this path.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
for _, t := range reqTokens {
if tokenResult[t] == 404 {
// The client provided valid token(s), but the
// collection was not found.
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
return
}
}
// data that has been deleted. Allow a referrer to
// provide this context somehow?
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- w.WriteHeader(http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
openPath := "/" + strings.Join(targetPath, "/")
if f, err := fs.Open(openPath); os.IsNotExist(err) {
// Requested non-existent path
- w.WriteHeader(http.StatusNotFound)
+ http.Error(w, notFoundMessage, http.StatusNotFound)
} else if err != nil {
// Some other (unexpected) error
http.Error(w, "open: "+err.Error(), http.StatusInternalServerError)
func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []string, credentialsOK, attachment bool) {
if len(tokens) == 0 {
w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
- http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
+ http.Error(w, unauthorizedMessage, http.StatusUnauthorized)
return
}
if writeMethod[r.Method] {
Value: auth.EncodeTokenCookie([]byte(formToken)),
Path: "/",
HttpOnly: true,
+ SameSite: http.SameSiteLaxMode,
})
}
}
s.testServer.Handler.ServeHTTP(resp, req)
c.Check(resp.Code, check.Equals, http.StatusNotFound)
- c.Check(resp.Body.String(), check.Equals, "")
+ c.Check(resp.Body.String(), check.Equals, notFoundMessage+"\n")
}
}
// depending on the authz method.
c.Check(code, check.Equals, failCode)
}
- c.Check(body, check.Equals, "")
+ if code == 404 {
+ c.Check(body, check.Equals, notFoundMessage+"\n")
+ } else {
+ c.Check(body, check.Equals, unauthorizedMessage+"\n")
+ }
}
}
}
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
"",
"",
http.StatusUnauthorized,
- "",
+ unauthorizedMessage+"\n",
)
}
"application/x-www-form-urlencoded",
url.Values{"api_token": {arvadostest.SpectatorToken}}.Encode(),
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
"",
"",
http.StatusNotFound,
- "",
+ notFoundMessage+"\n",
)
}
c.Check(resp.Code, check.Equals, http.StatusOK)
c.Check(resp.Body.String(), check.Equals, "foo")
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
+
+ // GET + Origin header is representative of both AJAX GET
+ // requests and inline images via <IMG crossorigin="anonymous"
+ // src="...">.
+ u.RawQuery = "api_token=" + url.QueryEscape(arvadostest.ActiveTokenV2)
+ req = &http.Request{
+ Method: "GET",
+ Host: u.Host,
+ URL: u,
+ RequestURI: u.RequestURI(),
+ Header: http.Header{
+ "Origin": {"https://origin.example"},
+ },
+ }
+ resp = httptest.NewRecorder()
+ s.testServer.Handler.ServeHTTP(resp, req)
+ c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Check(resp.Body.String(), check.Equals, "foo")
+ c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, "*")
}
func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, hostPath, queryString, contentType, reqBody string, expectStatus int, expectRespBody string) *httptest.ResponseRecorder {
package main
import (
+ "crypto/hmac"
+ "crypto/sha256"
"encoding/xml"
"errors"
"fmt"
+ "hash"
"io"
"net/http"
+ "net/url"
"os"
"path/filepath"
+ "regexp"
"sort"
"strconv"
"strings"
+ "time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/AdRoll/goamz/s3"
)
-const s3MaxKeys = 1000
+const (
+ s3MaxKeys = 1000
+ s3SignAlgorithm = "AWS4-HMAC-SHA256"
+ s3MaxClockSkew = 5 * time.Minute
+)
+
+func hmacstring(msg string, key []byte) []byte {
+ h := hmac.New(sha256.New, key)
+ io.WriteString(h, msg)
+ return h.Sum(nil)
+}
+
+func hashdigest(h hash.Hash, payload string) string {
+ io.WriteString(h, payload)
+ return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// Signing key for given secret key and request attrs.
+func s3signatureKey(key, datestamp, regionName, serviceName string) []byte {
+ return hmacstring("aws4_request",
+ hmacstring(serviceName,
+ hmacstring(regionName,
+ hmacstring(datestamp, []byte("AWS4"+key)))))
+}
+
+// Canonical query string for S3 V4 signature: sorted keys, spaces
+// escaped as %20 instead of +, keyvalues joined with &.
+func s3querystring(u *url.URL) string {
+ keys := make([]string, 0, len(u.Query()))
+ values := make(map[string]string, len(u.Query()))
+ for k, vs := range u.Query() {
+ k = strings.Replace(url.QueryEscape(k), "+", "%20", -1)
+ keys = append(keys, k)
+ for _, v := range vs {
+ v = strings.Replace(url.QueryEscape(v), "+", "%20", -1)
+ if values[k] != "" {
+ values[k] += "&"
+ }
+ values[k] += k + "=" + v
+ }
+ }
+ sort.Strings(keys)
+ for i, k := range keys {
+ keys[i] = values[k]
+ }
+ return strings.Join(keys, "&")
+}
+
+var reMultipleSlashChars = regexp.MustCompile(`//+`)
+
+func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string, error) {
+ timefmt, timestr := "20060102T150405Z", r.Header.Get("X-Amz-Date")
+ if timestr == "" {
+ timefmt, timestr = time.RFC1123, r.Header.Get("Date")
+ }
+ t, err := time.Parse(timefmt, timestr)
+ if err != nil {
+ return "", fmt.Errorf("invalid timestamp %q: %s", timestr, err)
+ }
+ if skew := time.Now().Sub(t); skew < -s3MaxClockSkew || skew > s3MaxClockSkew {
+ return "", errors.New("exceeded max clock skew")
+ }
+
+ var canonicalHeaders string
+ for _, h := range strings.Split(signedHeaders, ";") {
+ if h == "host" {
+ canonicalHeaders += h + ":" + r.Host + "\n"
+ } else {
+ canonicalHeaders += h + ":" + r.Header.Get(h) + "\n"
+ }
+ }
+
+ normalizedURL := *r.URL
+ normalizedURL.RawPath = ""
+ normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+ canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
+ ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
+ return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
+}
+
+func s3signature(secretKey, scope, signedHeaders, stringToSign string) (string, error) {
+ // scope is {datestamp}/{region}/{service}/aws4_request
+ drs := strings.Split(scope, "/")
+ if len(drs) != 4 {
+ return "", fmt.Errorf("invalid scope %q", scope)
+ }
+ key := s3signatureKey(secretKey, drs[0], drs[1], drs[2])
+ return hashdigest(hmac.New(sha256.New, key), stringToSign), nil
+}
+
+var v2tokenUnderscore = regexp.MustCompile(`^v2_[a-z0-9]{5}-gj3su-[a-z0-9]{15}_`)
+
+func unescapeKey(key string) string {
+ if v2tokenUnderscore.MatchString(key) {
+ // Entire Arvados token, with "/" replaced by "_" to
+ // avoid colliding with the Authorization header
+ // format.
+ return strings.Replace(key, "_", "/", -1)
+ } else if s, err := url.PathUnescape(key); err == nil {
+ return s
+ } else {
+ return key
+ }
+}
+
+// checks3signature verifies the given S3 V4 signature and returns the
+// Arvados token that corresponds to the given accessKey. An error is
+// returned if accessKey is not a valid token UUID or the signature
+// does not match.
+func (h *handler) checks3signature(r *http.Request) (string, error) {
+ var key, scope, signedHeaders, signature string
+ authstring := strings.TrimPrefix(r.Header.Get("Authorization"), s3SignAlgorithm+" ")
+ for _, cmpt := range strings.Split(authstring, ",") {
+ cmpt = strings.TrimSpace(cmpt)
+ split := strings.SplitN(cmpt, "=", 2)
+ switch {
+ case len(split) != 2:
+ // (?) ignore
+ case split[0] == "Credential":
+ keyandscope := strings.SplitN(split[1], "/", 2)
+ if len(keyandscope) == 2 {
+ key, scope = keyandscope[0], keyandscope[1]
+ }
+ case split[0] == "SignedHeaders":
+ signedHeaders = split[1]
+ case split[0] == "Signature":
+ signature = split[1]
+ }
+ }
+
+ client := (&arvados.Client{
+ APIHost: h.Config.cluster.Services.Controller.ExternalURL.Host,
+ Insecure: h.Config.cluster.TLS.Insecure,
+ }).WithRequestID(r.Header.Get("X-Request-Id"))
+ var aca arvados.APIClientAuthorization
+ var secret string
+ var err error
+ if len(key) == 27 && key[5:12] == "-gj3su-" {
+ // Access key is the UUID of an Arvados token, secret
+ // key is the secret part.
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+h.Config.cluster.SystemRootToken)
+ err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/"+key, nil, nil)
+ secret = aca.APIToken
+ } else {
+ // Access key and secret key are both an entire
+ // Arvados token or OIDC access token.
+ ctx := arvados.ContextWithAuthorization(r.Context(), "Bearer "+unescapeKey(key))
+ err = client.RequestAndDecodeContext(ctx, &aca, "GET", "arvados/v1/api_client_authorizations/current", nil, nil)
+ secret = key
+ }
+ if err != nil {
+ ctxlog.FromContext(r.Context()).WithError(err).WithField("UUID", key).Info("token lookup failed")
+ return "", errors.New("invalid access key")
+ }
+ stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, r)
+ if err != nil {
+ return "", err
+ }
+ expect, err := s3signature(secret, scope, signedHeaders, stringToSign)
+ if err != nil {
+ return "", err
+ } else if expect != signature {
+ return "", fmt.Errorf("signature does not match (scope %q signedHeaders %q stringToSign %q)", scope, signedHeaders, stringToSign)
+ }
+ return aca.TokenV2(), nil
+}
+
+func s3ErrorResponse(w http.ResponseWriter, s3code string, message string, resource string, code int) {
+ w.Header().Set("Content-Type", "application/xml")
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ w.WriteHeader(code)
+ var errstruct struct {
+ Code string
+ Message string
+ Resource string
+ RequestId string
+ }
+ errstruct.Code = s3code
+ errstruct.Message = message
+ errstruct.Resource = resource
+ errstruct.RequestId = ""
+ enc := xml.NewEncoder(w)
+ fmt.Fprint(w, xml.Header)
+ enc.EncodeElement(errstruct, xml.StartElement{Name: xml.Name{Local: "Error"}})
+}
+
+var NoSuchKey = "NoSuchKey"
+var NoSuchBucket = "NoSuchBucket"
+var InvalidArgument = "InvalidArgument"
+var InternalError = "InternalError"
+var UnauthorizedAccess = "UnauthorizedAccess"
+var InvalidRequest = "InvalidRequest"
+var SignatureDoesNotMatch = "SignatureDoesNotMatch"
// serveS3 handles r and returns true if r is a request from an S3
// client, otherwise it returns false.
if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "AWS ") {
split := strings.SplitN(auth[4:], ":", 2)
if len(split) < 2 {
- w.WriteHeader(http.StatusUnauthorized)
+ s3ErrorResponse(w, InvalidRequest, "malformed Authorization header", r.URL.Path, http.StatusUnauthorized)
return true
}
- token = split[0]
- } else if strings.HasPrefix(auth, "AWS4-HMAC-SHA256 ") {
- for _, cmpt := range strings.Split(auth[17:], ",") {
- cmpt = strings.TrimSpace(cmpt)
- split := strings.SplitN(cmpt, "=", 2)
- if len(split) == 2 && split[0] == "Credential" {
- keyandscope := strings.Split(split[1], "/")
- if len(keyandscope[0]) > 0 {
- token = keyandscope[0]
- break
- }
- }
- }
- if token == "" {
- w.WriteHeader(http.StatusBadRequest)
- fmt.Println(w, "invalid V4 signature")
+ token = unescapeKey(split[0])
+ } else if strings.HasPrefix(auth, s3SignAlgorithm+" ") {
+ t, err := h.checks3signature(r)
+ if err != nil {
+ s3ErrorResponse(w, SignatureDoesNotMatch, "signature verification failed: "+err.Error(), r.URL.Path, http.StatusForbidden)
return true
}
+ token = t
} else {
return false
}
_, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
if err != nil {
- http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
defer release()
fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
- objectNameGiven := strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ var objectNameGiven bool
+ var bucketName string
+ fspath := "/by_id"
+ if id := parseCollectionIDFromDNSName(r.Host); id != "" {
+ fspath += "/" + id
+ bucketName = id
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 0
+ } else {
+ bucketName = strings.SplitN(strings.TrimPrefix(r.URL.Path, "/"), "/", 2)[0]
+ objectNameGiven = strings.Count(strings.TrimSuffix(r.URL.Path, "/"), "/") > 1
+ }
+ fspath += reMultipleSlashChars.ReplaceAllString(r.URL.Path, "/")
switch {
case r.Method == http.MethodGet && !objectNameGiven:
fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
} else {
// ListObjects
- h.s3list(w, r, fs)
+ h.s3list(bucketName, w, r, fs)
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
- fspath := "/by_id" + r.URL.Path
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
if err == nil && fi.IsDir() {
w.WriteHeader(http.StatusOK)
} else if os.IsNotExist(err) {
- w.WriteHeader(http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchBucket, "The specified bucket does not exist.", r.URL.Path, http.StatusNotFound)
} else {
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
}
return true
}
if os.IsNotExist(err) ||
(err != nil && err.Error() == "not a directory") ||
(fi != nil && fi.IsDir()) {
- http.Error(w, "not found", http.StatusNotFound)
+ s3ErrorResponse(w, NoSuchKey, "The specified key does not exist.", r.URL.Path, http.StatusNotFound)
return true
}
// shallow copy r, and change URL path
return true
case r.Method == http.MethodPut:
if !objectNameGiven {
- http.Error(w, "missing object name in PUT request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
var objectIsDir bool
if strings.HasSuffix(fspath, "/") {
if !h.Config.cluster.Collections.S3FolderObjects {
- http.Error(w, "invalid object name: trailing slash", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "invalid object name: trailing slash", r.URL.Path, http.StatusBadRequest)
return true
}
n, err := r.Body.Read(make([]byte, 1))
if err != nil && err != io.EOF {
- http.Error(w, fmt.Sprintf("error reading request body: %s", err), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, fmt.Sprintf("error reading request body: %s", err), r.URL.Path, http.StatusInternalServerError)
return true
} else if n > 0 {
- http.Error(w, "cannot create object with trailing '/' char unless content is empty", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless content is empty", r.URL.Path, http.StatusBadRequest)
return true
} else if strings.SplitN(r.Header.Get("Content-Type"), ";", 2)[0] != "application/x-directory" {
- http.Error(w, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "cannot create object with trailing '/' char unless Content-Type is 'application/x-directory'", r.URL.Path, http.StatusBadRequest)
return true
}
// Given PUT "foo/bar/", we'll use "foo/bar/."
fi, err := fs.Stat(fspath)
if err != nil && err.Error() == "not a directory" {
// requested foo/bar, but foo is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
if strings.HasSuffix(r.URL.Path, "/") && err == nil && !fi.IsDir() {
// requested foo/bar/, but foo/bar is a file
- http.Error(w, "object name conflicts with existing object", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "object name conflicts with existing object", r.URL.Path, http.StatusBadRequest)
return true
}
// create missing parent/intermediate directories, if any
dir := fspath[:i]
if strings.HasSuffix(dir, "/") {
err = errors.New("invalid object name (consecutive '/' chars)")
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Mkdir(dir, 0755)
// Cannot create a directory
// here.
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
} else if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir %q failed: %w", dir, err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
}
}
if err != nil {
err = fmt.Errorf("open %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
defer f.Close()
_, err = io.Copy(f, r.Body)
if err != nil {
err = fmt.Errorf("write to %q failed: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
err = f.Close()
if err != nil {
err = fmt.Errorf("write to %q failed: close: %w", r.URL.Path, err)
- http.Error(w, err.Error(), http.StatusBadGateway)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusBadGateway)
return true
}
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
if !objectNameGiven || r.URL.Path == "/" {
- http.Error(w, "missing object name in DELETE request", http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
}
- fspath := "by_id" + r.URL.Path
if strings.HasSuffix(fspath, "/") {
fspath = strings.TrimSuffix(fspath, "/")
fi, err := fs.Stat(fspath)
w.WriteHeader(http.StatusNoContent)
return true
} else if err != nil {
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
} else if !fi.IsDir() {
// if "foo" exists and is a file, then
}
if err != nil {
err = fmt.Errorf("rm failed: %w", err)
- http.Error(w, err.Error(), http.StatusBadRequest)
+ s3ErrorResponse(w, InvalidArgument, err.Error(), r.URL.Path, http.StatusBadRequest)
return true
}
err = fs.Sync()
if err != nil {
err = fmt.Errorf("sync failed: %w", err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
w.WriteHeader(http.StatusNoContent)
return true
default:
- http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed)
+
return true
}
}
var errDone = errors.New("done")
-func (h *handler) s3list(w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
+func (h *handler) s3list(bucket string, w http.ResponseWriter, r *http.Request, fs arvados.CustomFileSystem) {
var params struct {
- bucket string
delimiter string
marker string
maxKeys int
prefix string
}
- params.bucket = strings.SplitN(r.URL.Path[1:], "/", 2)[0]
params.delimiter = r.FormValue("delimiter")
params.marker = r.FormValue("marker")
if mk, _ := strconv.ParseInt(r.FormValue("max-keys"), 10, 64); mk > 0 && mk < s3MaxKeys {
}
params.prefix = r.FormValue("prefix")
- bucketdir := "by_id/" + params.bucket
+ bucketdir := "by_id/" + bucket
// walkpath is the directory (relative to bucketdir) we need
// to walk: the innermost directory that is guaranteed to
// contain all paths that have the requested prefix. Examples:
}
resp := listResp{
ListResp: s3.ListResp{
- Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+ Name: bucket,
Prefix: params.prefix,
Delimiter: params.delimiter,
Marker: params.marker,
import (
"bytes"
"crypto/rand"
+ "crypto/sha256"
"fmt"
"io/ioutil"
"net/http"
+ "net/http/httptest"
+ "net/url"
"os"
+ "os/exec"
"strings"
"sync"
"time"
err = arv.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
c.Assert(err, check.IsNil)
- auth := aws.NewAuth(arvadostest.ActiveTokenV2, arvadostest.ActiveTokenV2, "", time.Now().Add(time.Hour))
+ auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
region := aws.Region{
Name: s.testServer.Addr,
S3Endpoint: "http://" + s.testServer.Addr,
}
client := s3.New(*auth, region)
+ client.Signature = aws.V4Signature
return s3stage{
arv: arv,
ac: ac,
}
}
+func (s *IntegrationSuite) TestS3Signatures(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ bucket := stage.collbucket
+ for _, trial := range []struct {
+ success bool
+ signature int
+ accesskey string
+ secretkey string
+ }{
+ {true, aws.V2Signature, arvadostest.ActiveToken, "none"},
+ {true, aws.V2Signature, url.QueryEscape(arvadostest.ActiveTokenV2), "none"},
+ {true, aws.V2Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), "none"},
+ {false, aws.V2Signature, "none", "none"},
+ {false, aws.V2Signature, "none", arvadostest.ActiveToken},
+
+ {true, aws.V4Signature, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken},
+ {true, aws.V4Signature, arvadostest.ActiveToken, arvadostest.ActiveToken},
+ {true, aws.V4Signature, url.QueryEscape(arvadostest.ActiveTokenV2), url.QueryEscape(arvadostest.ActiveTokenV2)},
+ {true, aws.V4Signature, strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1), strings.Replace(arvadostest.ActiveTokenV2, "/", "_", -1)},
+ {false, aws.V4Signature, arvadostest.ActiveToken, ""},
+ {false, aws.V4Signature, arvadostest.ActiveToken, "none"},
+ {false, aws.V4Signature, "none", arvadostest.ActiveToken},
+ {false, aws.V4Signature, "none", "none"},
+ } {
+ c.Logf("%#v", trial)
+ bucket.S3.Auth = *(aws.NewAuth(trial.accesskey, trial.secretkey, "", time.Now().Add(time.Hour)))
+ bucket.S3.Signature = trial.signature
+ _, err := bucket.GetReader("emptyfile")
+ if trial.success {
+ c.Check(err, check.IsNil)
+ } else {
+ c.Check(err, check.NotNil)
+ }
+ }
+}
+
func (s *IntegrationSuite) TestS3HeadBucket(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
// GetObject
rdr, err = bucket.GetReader(prefix + "missingfile")
- c.Check(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`)
// HeadObject
exists, err := bucket.Exists(prefix + "missingfile")
c.Check(err, check.IsNil)
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
c.Check(resp.ContentLength, check.Equals, int64(4))
+
+ // HeadObject with superfluous leading slashes
+ exists, err = bucket.Exists(prefix + "//sailboat.txt")
+ c.Check(err, check.IsNil)
+ c.Check(exists, check.Equals, true)
}
func (s *IntegrationSuite) TestS3CollectionPutObjectSuccess(c *check.C) {
path: "newdir/newfile",
size: 1 << 26,
contentType: "application/octet-stream",
+ }, {
+ path: "/aaa",
+ size: 2,
+ contentType: "application/octet-stream",
+ }, {
+ path: "//bbb",
+ size: 2,
+ contentType: "application/octet-stream",
+ }, {
+ path: "ccc//",
+ size: 0,
+ contentType: "application/x-directory",
}, {
path: "newdir1/newdir2/newfile",
size: 0,
objname := prefix + trial.path
_, err := bucket.GetReader(objname)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ if !c.Check(err, check.NotNil) {
+ continue
+ }
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ if !c.Check(err, check.ErrorMatches, `The specified key does not exist.`) {
+ continue
+ }
buf := make([]byte, trial.size)
rand.Read(buf)
c.Logf("=== %v", trial)
_, err := bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
buf := make([]byte, trial.size)
rand.Read(buf)
err = bucket.PutReader(trial.path, bytes.NewReader(buf), int64(len(buf)), trial.contentType, s3.Private, s3.Options{})
- c.Check(err, check.ErrorMatches, `400 Bad Request`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 400)
+ c.Check(err.(*s3.Error).Code, check.Equals, `InvalidArgument`)
+ c.Check(err, check.ErrorMatches, `(mkdir "/by_id/zzzzz-j7d0g-[a-z0-9]{15}/newdir2?"|open "/zzzzz-j7d0g-[a-z0-9]{15}/newfile") failed: invalid argument`)
_, err = bucket.GetReader(trial.path)
- c.Assert(err, check.ErrorMatches, `404 Not Found`)
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Assert(err, check.ErrorMatches, `The specified key does not exist.`)
}
}
}
func (s *IntegrationSuite) testS3PutObjectFailure(c *check.C, bucket *s3.Bucket, prefix string) {
s.testServer.Config.cluster.Collections.S3FolderObjects = false
+
var wg sync.WaitGroup
for _, trial := range []struct {
path string
path: "/",
}, {
path: "//",
- }, {
- path: "foo//bar",
}, {
path: "",
},
rand.Read(buf)
err := bucket.PutReader(objname, bytes.NewReader(buf), int64(len(buf)), "application/octet-stream", s3.Private, s3.Options{})
- if !c.Check(err, check.ErrorMatches, `400 Bad.*`, check.Commentf("PUT %q should fail", objname)) {
+ if !c.Check(err, check.ErrorMatches, `(invalid object name.*|open ".*" failed.*|object name conflicts with existing object|Missing object name in PUT request.)`, check.Commentf("PUT %q should fail", objname)) {
return
}
if objname != "" && objname != "/" {
_, err = bucket.GetReader(objname)
- c.Check(err, check.ErrorMatches, `404 Not Found`, check.Commentf("GET %q should return 404", objname))
+ c.Check(err.(*s3.Error).StatusCode, check.Equals, 404)
+ c.Check(err.(*s3.Error).Code, check.Equals, `NoSuchKey`)
+ c.Check(err, check.ErrorMatches, `The specified key does not exist.`, check.Commentf("GET %q should return 404", objname))
}
}()
}
c.Assert(fs.Sync(), check.IsNil)
}
+func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
+ scope := "20200202/region/service/aws4_request"
+ signedHeaders := "date"
+ req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
+ stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
+ c.Assert(err, check.IsNil)
+ sig, err := s3signature(secret, scope, signedHeaders, stringToSign)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", s3SignAlgorithm+" Credential="+key+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
+}
+
+func (s *IntegrationSuite) TestS3VirtualHostStyleRequests(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ url string
+ method string
+ body string
+ responseCode int
+ responseRegexp []string
+ }{
+ {
+ url: "https://" + stage.collbucket.Name + ".example.com/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + strings.Replace(stage.coll.PortableDataHash, "+", "-", -1) + ".example.com/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/?prefix=" + stage.coll.Name + "/&delimiter=/",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`(?ms).*sailboat\.txt.*`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/sailboat.txt",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`⛵\n`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+ method: "PUT",
+ body: "boop",
+ responseCode: http.StatusOK,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "/beep",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`boop`},
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "GET",
+ responseCode: http.StatusNotFound,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "PUT",
+ body: "boop",
+ responseCode: http.StatusOK,
+ },
+ {
+ url: "https://" + stage.projbucket.Name + ".example.com/" + stage.coll.Name + "//boop",
+ method: "GET",
+ responseCode: http.StatusOK,
+ responseRegexp: []string{`boop`},
+ },
+ } {
+ url, err := url.Parse(trial.url)
+ c.Assert(err, check.IsNil)
+ req, err := http.NewRequest(trial.method, url.String(), bytes.NewReader([]byte(trial.body)))
+ c.Assert(err, check.IsNil)
+ s.sign(c, req, arvadostest.ActiveTokenUUID, arvadostest.ActiveToken)
+ rr := httptest.NewRecorder()
+ s.testServer.Server.Handler.ServeHTTP(rr, req)
+ resp := rr.Result()
+ c.Check(resp.StatusCode, check.Equals, trial.responseCode)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ for _, re := range trial.responseRegexp {
+ c.Check(string(body), check.Matches, re)
+ }
+ }
+}
+
+func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ rawPath string
+ normalizedPath string
+ }{
+ {"/foo", "/foo"}, // boring case
+ {"/foo%5fbar", "/foo_bar"}, // _ must not be escaped
+ {"/foo%2fbar", "/foo/bar"}, // / must not be escaped
+ {"/(foo)", "/%28foo%29"}, // () must be escaped
+ {"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
+ } {
+ date := time.Now().UTC().Format("20060102T150405Z")
+ scope := "20200202/fakeregion/S3/aws4_request"
+ canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
+ c.Logf("canonicalRequest %q", canonicalRequest)
+ expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
+ c.Logf("expected stringToSign %q", expect)
+
+ req, err := http.NewRequest("GET", "https://host.example.com"+trial.rawPath, nil)
+ req.Header.Set("X-Amz-Date", date)
+ req.Host = "host.example.com"
+
+ obtained, err := s3stringToSign(s3SignAlgorithm, scope, "host", req)
+ if !c.Check(err, check.IsNil) {
+ continue
+ }
+ c.Check(obtained, check.Equals, expect)
+ }
+}
+
func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
c.Logf("=== trial %+v keys %q prefixes %q nextMarker %q", trial, gotKeys, gotPrefixes, resp.NextMarker)
}
}
+
+// TestS3cmd checks compatibility with the s3cmd command line tool, if
+// it's installed. As of Debian buster, s3cmd is only in backports, so
+// `arvados-server install` don't install it, and this test skips if
+// it's not installed.
+func (s *IntegrationSuite) TestS3cmd(c *check.C) {
+ if _, err := exec.LookPath("s3cmd"); err != nil {
+ c.Skip("s3cmd not found")
+ return
+ }
+
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ cmd := exec.Command("s3cmd", "--no-ssl", "--host="+s.testServer.Addr, "--host-bucket="+s.testServer.Addr, "--access_key="+arvadostest.ActiveTokenUUID, "--secret_key="+arvadostest.ActiveToken, "ls", "s3://"+arvadostest.FooCollection)
+ buf, err := cmd.CombinedOutput()
+ c.Check(err, check.IsNil)
+ c.Check(string(buf), check.Matches, `.* 3 +s3://`+arvadostest.FooCollection+`/foo\n`)
+}
+
+func (s *IntegrationSuite) TestS3BucketInHost(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ hdr, body, _ := s.runCurl(c, "AWS "+arvadostest.ActiveTokenV2+":none", stage.coll.UUID+".collections.example.com", "/sailboat.txt")
+ c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
+ c.Check(body, check.Equals, "⛵\n")
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "io/ioutil"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/aws/defaults"
+ "github.com/aws/aws-sdk-go-v2/aws/ec2metadata"
+ "github.com/aws/aws-sdk-go-v2/aws/ec2rolecreds"
+ "github.com/aws/aws-sdk-go-v2/aws/endpoints"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ check "gopkg.in/check.v1"
+)
+
+func (s *IntegrationSuite) TestS3AWSSDK(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ cfg := defaults.Config()
+ cfg.Credentials = aws.NewChainProvider([]aws.CredentialsProvider{
+ aws.NewStaticCredentialsProvider(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, ""),
+ ec2rolecreds.New(ec2metadata.New(cfg)),
+ })
+ cfg.EndpointResolver = aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+ if service == "s3" {
+ return aws.Endpoint{
+ URL: "http://" + s.testServer.Addr,
+ SigningRegion: "custom-signing-region",
+ }, nil
+ }
+ return endpoints.NewDefaultResolver().ResolveEndpoint(service, region)
+ })
+ client := s3.New(cfg)
+ client.ForcePathStyle = true
+ listreq := client.ListObjectsV2Request(&s3.ListObjectsV2Input{
+ Bucket: aws.String(arvadostest.FooCollection),
+ MaxKeys: aws.Int64(100),
+ Prefix: aws.String(""),
+ ContinuationToken: nil,
+ })
+ resp, err := listreq.Send(context.Background())
+ c.Assert(err, check.IsNil)
+ c.Check(resp.Contents, check.HasLen, 1)
+ for _, key := range resp.Contents {
+ c.Check(*key.Key, check.Equals, "foo")
+ }
+
+ p := make([]byte, 100000000)
+ for i := range p {
+ p[i] = byte('a')
+ }
+ putreq := client.PutObjectRequest(&s3.PutObjectInput{
+ Body: bytes.NewReader(p),
+ Bucket: aws.String(stage.collbucket.Name),
+ ContentType: aws.String("application/octet-stream"),
+ Key: aws.String("aaaa"),
+ })
+ _, err = putreq.Send(context.Background())
+ c.Assert(err, check.IsNil)
+
+ getreq := client.GetObjectRequest(&s3.GetObjectInput{
+ Bucket: aws.String(stage.collbucket.Name),
+ Key: aws.String("aaaa"),
+ })
+ getresp, err := getreq.Send(context.Background())
+ c.Assert(err, check.IsNil)
+ getdata, err := ioutil.ReadAll(getresp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(bytes.Equal(getdata, p), check.Equals, true)
+}
} {
hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
if token != "" {
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
- c.Check(body, check.Equals, "")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
if len(body) > 0 {
- c.Check(body, check.Equals, "404 page not found\n")
+ c.Check(body, check.Equals, notFoundMessage+"\n")
}
}
}
}
// Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
curlArgs := []string{"--silent", "--show-error", "--include"}
testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
- if token != "" {
- curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
+ if strings.Contains(auth, " ") {
+ // caller supplied entire Authorization header value
+ curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
+ } else if auth != "" {
+ // caller supplied Arvados token
+ curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
}
curlArgs = append(curlArgs, args...)
curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
cfg.cluster.ManagementToken = arvadostest.ManagementToken
+ cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
s.testServer = &server{Config: cfg}
err = s.testServer.Start(ctxlog.TestLogger(c))
return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
}
-type ApiTokenCache struct {
+type APITokenCache struct {
tokens map[string]int64
lock sync.Mutex
expireTime int64
}
-// Cache the token and set an expire time. If we already have an expire time
-// on the token, it is not updated.
-func (this *ApiTokenCache) RememberToken(token string) {
- this.lock.Lock()
- defer this.lock.Unlock()
+// RememberToken caches the token and set an expire time. If we already have
+// an expire time on the token, it is not updated.
+func (cache *APITokenCache) RememberToken(token string) {
+ cache.lock.Lock()
+ defer cache.lock.Unlock()
now := time.Now().Unix()
- if this.tokens[token] == 0 {
- this.tokens[token] = now + this.expireTime
+ if cache.tokens[token] == 0 {
+ cache.tokens[token] = now + cache.expireTime
}
}
-// Check if the cached token is known and still believed to be valid.
-func (this *ApiTokenCache) RecallToken(token string) bool {
- this.lock.Lock()
- defer this.lock.Unlock()
+// RecallToken checks if the cached token is known and still believed to be
+// valid.
+func (cache *APITokenCache) RecallToken(token string) bool {
+ cache.lock.Lock()
+ defer cache.lock.Unlock()
now := time.Now().Unix()
- if this.tokens[token] == 0 {
+ if cache.tokens[token] == 0 {
// Unknown token
return false
- } else if now < this.tokens[token] {
+ } else if now < cache.tokens[token] {
// Token is known and still valid
return true
} else {
// Token is expired
- this.tokens[token] = 0
+ cache.tokens[token] = 0
return false
}
}
+// GetRemoteAddress returns a string with the remote address for the request.
+// If the X-Forwarded-For header is set and has a non-zero length, it returns a
+// string made from a comma separated list of all the remote addresses,
+// starting with the one(s) from the X-Forwarded-For header.
func GetRemoteAddress(req *http.Request) string {
if xff := req.Header.Get("X-Forwarded-For"); xff != "" {
return xff + "," + req.RemoteAddr
return req.RemoteAddr
}
-func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *ApiTokenCache, req *http.Request) (pass bool, tok string) {
+func CheckAuthorizationHeader(kc *keepclient.KeepClient, cache *APITokenCache, req *http.Request) (pass bool, tok string) {
parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2)
if len(parts) < 2 || !(parts[0] == "OAuth2" || parts[0] == "Bearer") || len(parts[1]) == 0 {
return false, ""
type proxyHandler struct {
http.Handler
*keepclient.KeepClient
- *ApiTokenCache
+ *APITokenCache
timeout time.Duration
transport *http.Transport
}
KeepClient: kc,
timeout: timeout,
transport: &transport,
- ApiTokenCache: &ApiTokenCache{
+ APITokenCache: &APITokenCache{
tokens: make(map[string]int64),
expireTime: 300,
},
SetCorsHeaders(resp)
}
-var BadAuthorizationHeader = errors.New("Missing or invalid Authorization header")
-var ContentLengthMismatch = errors.New("Actual length != expected content length")
-var MethodNotSupported = errors.New("Method not supported")
+var errBadAuthorizationHeader = errors.New("Missing or invalid Authorization header")
+var errContentLengthMismatch = errors.New("Actual length != expected content length")
+var errMethodNotSupported = errors.New("Method not supported")
var removeHint, _ = regexp.Compile("\\+K@[a-z0-9]{5}(\\+|$)")
var pass bool
var tok string
- if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass {
- status, err = http.StatusForbidden, BadAuthorizationHeader
+ if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass {
+ status, err = http.StatusForbidden, errBadAuthorizationHeader
return
}
defer reader.Close()
}
default:
- status, err = http.StatusNotImplemented, MethodNotSupported
+ status, err = http.StatusNotImplemented, errMethodNotSupported
return
}
case "GET":
responseLength, err = io.Copy(resp, reader)
if err == nil && expectLength > -1 && responseLength != expectLength {
- err = ContentLengthMismatch
+ err = errContentLengthMismatch
}
}
case keepclient.Error:
}
}
-var LengthRequiredError = errors.New(http.StatusText(http.StatusLengthRequired))
-var LengthMismatchError = errors.New("Locator size hint does not match Content-Length header")
+var errLengthRequired = errors.New(http.StatusText(http.StatusLengthRequired))
+var errLengthMismatch = errors.New("Locator size hint does not match Content-Length header")
func (h *proxyHandler) Put(resp http.ResponseWriter, req *http.Request) {
if err := h.checkLoop(resp, req); err != nil {
_, err = fmt.Sscanf(req.Header.Get("Content-Length"), "%d", &expectLength)
if err != nil || expectLength < 0 {
- err = LengthRequiredError
+ err = errLengthRequired
status = http.StatusLengthRequired
return
}
status = http.StatusBadRequest
return
} else if loc.Size > 0 && int64(loc.Size) != expectLength {
- err = LengthMismatchError
+ err = errLengthMismatch
status = http.StatusBadRequest
return
}
var pass bool
var tok string
- if pass, tok = CheckAuthorizationHeader(kc, h.ApiTokenCache, req); !pass {
- err = BadAuthorizationHeader
+ if pass, tok = CheckAuthorizationHeader(kc, h.APITokenCache, req); !pass {
+ err = errBadAuthorizationHeader
status = http.StatusForbidden
return
}
// Check if the client specified the number of replicas
if req.Header.Get("X-Keep-Desired-Replicas") != "" {
var r int
- _, err := fmt.Sscanf(req.Header.Get(keepclient.X_Keep_Desired_Replicas), "%d", &r)
+ _, err := fmt.Sscanf(req.Header.Get(keepclient.XKeepDesiredReplicas), "%d", &r)
if err == nil {
kc.Want_replicas = r
}
}
// Tell the client how many successful PUTs we accomplished
- resp.Header().Set(keepclient.X_Keep_Replicas_Stored, fmt.Sprintf("%d", wroteReplicas))
+ resp.Header().Set(keepclient.XKeepReplicasStored, fmt.Sprintf("%d", wroteReplicas))
switch err.(type) {
case nil:
}()
kc := h.makeKeepClient(req)
- ok, token := CheckAuthorizationHeader(kc, h.ApiTokenCache, req)
+ ok, token := CheckAuthorizationHeader(kc, h.APITokenCache, req)
if !ok {
- status, err = http.StatusForbidden, BadAuthorizationHeader
+ status, err = http.StatusForbidden, errBadAuthorizationHeader
return
}
// Only GET method is supported
if req.Method != "GET" {
- status, err = http.StatusNotImplemented, MethodNotSupported
+ status, err = http.StatusNotImplemented, errMethodNotSupported
return
}
"time"
)
-// A Keep "block" is 64MB.
+// BlockSize for a Keep "block" is 64MB.
const BlockSize = 64 * 1024 * 1024
-// A Keep volume must have at least MinFreeKilobytes available
+// MinFreeKilobytes is the amount of space a Keep volume must have available
// in order to permit writes.
const MinFreeKilobytes = BlockSize / 1024
else
version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
end
+ version = version.sub("~dev", ".dev").sub("~rc", ".rc")
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.summary = "Set up local login accounts for Arvados users"
s.description = "Creates and updates local login accounts for Arvados users. Built from git commit #{git_hash}"
s.authors = ["Arvados Authors"]
- s.email = 'gem-dev@curoverse.com'
+ s.email = 'packaging@arvados.org'
s.licenses = ['AGPL-3.0']
s.files = ["bin/arvados-login-sync", "agpl-3.0.txt"]
s.executables << "arvados-login-sync"
//
// SPDX-License-Identifier: AGPL-3.0
-// Arvados-ws exposes Arvados APIs (currently just one, the
+// Package ws exposes Arvados APIs (currently just one, the
// cache-invalidation event feed at "ws://.../websocket") to
// websocket clients.
//
--publish=8900:8900
--publish=9000:9000
--publish=9002:9002
+ --publish=9004:9004
--publish=25101:25101
--publish=8001:8001
--publish=8002:8002
cd /usr/src/arvados/services/api
export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
export RAILS_ENV=development
-bundle exec rake db:drop
+flock $GEM_HOME/gems.lock bundle exec rake db:drop
rm $ARVADOS_CONTAINER_PATH/api_database_setup
rm $ARVADOS_CONTAINER_PATH/superuser_token
-rm $ARVADOS_CONTAINER_PATH/keep0-uuid
-rm $ARVADOS_CONTAINER_PATH/keep1-uuid
-rm $ARVADOS_CONTAINER_PATH/keepproxy-uuid
sv start api
sv start controller
sv start websockets
# put everything (/var/lib/arvados)
ENV ARVADOS_CONTAINER_PATH /var/lib/arvados-arvbox
+RUN /bin/ln -s /var/lib/arvados/bin/ruby /usr/local/bin/
+
# Start the supervisor.
ENV SVDIR /etc/service
STOPSIGNAL SIGINT
RUN echo "production" > $ARVADOS_CONTAINER_PATH/api_rails_env
RUN echo "production" > $ARVADOS_CONTAINER_PATH/workbench_rails_env
+# for the federation tests, the dev server watches a lot of files,
+# and we run three instances of the docker container. Bump up the
+# inotify limit from 8192, to avoid errors like
+# events.js:183
+# throw er; // Unhandled 'error' event
+# ^
+#
+# Error: watch /usr/src/workbench2/public ENOSPC
+# cf. https://github.com/facebook/jest/issues/3254
+RUN echo fs.inotify.max_user_watches=524288 >> /etc/sysctl.conf
+
RUN /usr/local/lib/arvbox/createusers.sh
RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- bundle exec rake db:setup
+ flock $GEM_HOME/gems.lock bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bundle exec rake db:migrate
WebDAVDownload:
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
- ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
+ ExternalURL: "https://$localip:${services[keep-web-dl-ssl]}/"
Composer:
ExternalURL: "https://$localip:${services[composer]}"
Controller:
password: ${database_pw}
dbname: arvados_${database_env}
client_encoding: utf8
- API:
- RailsSessionSecretToken: $secret_token
Collections:
BlobSigningKey: $blob_signing_key
DefaultReplication: 1
[arv-git-httpd]=9001
[keep-web]=9003
[keep-web-ssl]=9002
+ [keep-web-dl-ssl]=9004
[keepproxy]=25100
[keepproxy-ssl]=25101
[keepstore0]=25107
cd /usr/src/arvados
if [[ $UID = 0 ]] ; then
- /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go mod download
- if [[ ! -f /usr/local/bin/arvados-server ]]; then
- /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
- fi
-else
- flock /var/lib/gopath/gopath.lock go mod download
- if [[ ! -f /usr/local/bin/arvados-server ]]; then
- flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
- fi
+ RUNSU="/usr/local/lib/arvbox/runsu.sh"
+fi
+
+if [[ ! -f /usr/local/bin/arvados-server ]]; then
+ $RUNSU flock /var/lib/gopath/gopath.lock go mod download
+ $RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
+ $RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin
fi
-install $GOPATH/bin/arvados-server /usr/local/bin
mkdir -p $ARVADOS_CONTAINER_PATH/$1
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-set +e
-read -rd $'\000' keepservice <<EOF
-{
- "service_host":"localhost",
- "service_port":$2,
- "service_ssl_flag":false,
- "service_type":"disk"
-}
-EOF
-set -e
-
-if test -s $ARVADOS_CONTAINER_PATH/$1-uuid ; then
- keep_uuid=$(cat $ARVADOS_CONTAINER_PATH/$1-uuid)
- arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
-else
- UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
- echo $UUID > $ARVADOS_CONTAINER_PATH/$1-uuid
-fi
-
-management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token)
-
-set +e
-sv hup /var/lib/arvbox/service/keepproxy
-
-cat >$ARVADOS_CONTAINER_PATH/$1.yml <<EOF
-Listen: "localhost:$2"
-BlobSigningKeyFile: $ARVADOS_CONTAINER_PATH/blob_signing_key
-SystemAuthTokenFile: $ARVADOS_CONTAINER_PATH/superuser_token
-ManagementToken: $management_token
-MaxBuffers: 20
-EOF
-
-exec /usr/local/bin/keepstore -config=$ARVADOS_CONTAINER_PATH/$1.yml
+exec /usr/local/bin/keepstore
chown arvbox /dev/stderr
+# Load our custom sysctl.conf entries
+/sbin/sysctl -p >/dev/null
+
if test -z "$1" ; then
exec chpst -u arvbox:arvbox:docker $0-service
else
fi
run_bundler --without=development
-bundle exec passenger-config build-native-support
-bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
exit
fi
+touch $ARVADOS_CONTAINER_PATH/api.ready
+
exec bundle exec passenger start --port=${services[api]}
. /usr/local/lib/arvbox/common.sh
+if test "$1" != "--only-deps" ; then
+ while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
+ sleep 1
+ done
+fi
cd /usr/src/arvados/doc
run_bundler --without=development
fi
cd /usr/src/arvados/doc
-bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
+flock $GEM_HOME/gems.lock bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
. /usr/local/lib/arvbox/common.sh
+if test "$1" != "--only-deps" ; then
+ while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
+ sleep 1
+ done
+fi
+
mkdir -p $ARVADOS_CONTAINER_PATH/git
export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
EOF
while true ; do
- bundle exec script/arvados-git-sync.rb $RAILS_ENV
+ flock $GEM_HOME/gems.lock bundle exec script/arvados-git-sync.rb $RAILS_ENV
sleep 120
done
exit
fi
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-set +e
-read -rd $'\000' keepservice <<EOF
-{
- "service_host":"$localip",
- "service_port":${services[keepproxy-ssl]},
- "service_ssl_flag":true,
- "service_type":"proxy"
-}
-EOF
-set -e
-
-if test -s $ARVADOS_CONTAINER_PATH/keepproxy-uuid ; then
- keep_uuid=$(cat $ARVADOS_CONTAINER_PATH/keepproxy-uuid)
- arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
-else
- UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
- echo $UUID > $ARVADOS_CONTAINER_PATH/keepproxy-uuid
-fi
-
exec /usr/local/bin/keepproxy
proxy_redirect off;
}
}
+ server {
+ listen *:${services[keep-web-dl-ssl]} ssl default_server;
+ server_name keep-web-dl;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+ client_max_body_size 0;
+ location / {
+ proxy_pass http://keep-web;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_redirect off;
+ }
+ }
upstream keepproxy {
server localhost:${services[keepproxy]};
. /usr/local/lib/arvbox/common.sh
+if test "$1" != "--only-deps" ; then
+ while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
+ sleep 1
+ done
+fi
+
cd /usr/src/arvados/services/login-sync
run_bundler --binstubs=$PWD/binstubs
ln -sf /usr/src/arvados/services/login-sync/binstubs/arvados-login-sync /usr/local/bin/arvados-login-sync
. /usr/local/lib/arvbox/common.sh
+if test "$1" != "--only-deps" ; then
+ while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
+ sleep 1
+ done
+fi
+
cd /usr/src/arvados/apps/workbench
if test -s $ARVADOS_CONTAINER_PATH/workbench_rails_env ; then
fi
run_bundler --without=development
-bundle exec passenger-config build-native-support
-bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
if test "$1" = "--only-deps" ; then
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
rm config/application.yml
exit
fi
secret_token=$(cat $ARVADOS_CONTAINER_PATH/workbench_secret_token)
-if test -a /usr/src/arvados/apps/workbench/config/arvados_config.rb ; then
- rm -f config/application.yml
-else
-cat >config/application.yml <<EOF
-$RAILS_ENV:
- secret_token: $secret_token
- arvados_login_base: https://$localip:${services[controller-ssl]}/login
- arvados_v1_base: https://$localip:${services[controller-ssl]}/arvados/v1
- arvados_insecure_https: false
- keep_web_download_url: https://$localip:${services[keep-web-ssl]}/c=%{uuid_or_pdh}
- keep_web_url: https://$localip:${services[keep-web-ssl]}/c=%{uuid_or_pdh}
- arvados_docsite: http://$localip:${services[doc]}/
- force_ssl: false
- composer_url: http://$localip:${services[composer]}
- workbench2_url: https://$localip:${services[workbench2-ssl]}
-EOF
-
-(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
-fi
-
-RAILS_GROUPS=assets bundle exec rake npm:install
-bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
. /usr/local/lib/arvbox/common.sh
+if test "$1" != "--only-deps" ; then
+ while [ ! -f $ARVADOS_CONTAINER_PATH/api.ready ]; do
+ sleep 1
+ done
+fi
+
cd /usr/src/workbench2
npm -d install --prefix /usr/local --global yarn@1.17.3
# Add the arvados apt repository
echo "# apt.arvados.org" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
-echo "deb http://apt.arvados.org/ $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
+echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
# Add the arvados signing key
cat /tmp/1078ECD7.asc | $SUDO apt-key add -
import time
import os
import re
+import sys
SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
def choose_version_from():
- sdk_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', os.path.join(SETUP_DIR, "../../sdk/python")]).strip()
- cwl_ts = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', SETUP_DIR]).strip()
- if int(sdk_ts) > int(cwl_ts):
- getver = os.path.join(SETUP_DIR, "../../sdk/python")
- else:
- getver = SETUP_DIR
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
'--format=%H', curdir]).strip()
- myversion = subprocess.check_output([curdir+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
return myversion
def save_version(setup_dir, module, v):
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
def get_version(setup_dir, module):
env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
else:
try:
save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError):
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
pass
return read_version(setup_dir, module)
+++ /dev/null
-../../sdk/python/gittaggers.py
\ No newline at end of file
--- /dev/null
+[comment]: # (Copyright © The Arvados Authors. All rights reserved.)
+[comment]: # ()
+[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0)
+
+# Arvados install with Saltstack
+
+##### About
+
+This directory holds a small script to install Arvados on a single node, using the
+[Saltstack arvados-formula](https://github.com/saltstack-formulas/arvados-formula)
+in master-less mode.
+
+The fastest way to get it running is to modify the first lines in the `provision.sh`
+script to suit your needs, copy it in the host where you want to install Arvados
+and run it as root.
+
+There's an example `Vagrantfile` also, to install it in a vagrant box if you want
+to try it locally.
+
+For more information, please read https://doc.arvados.org/main/install/salt-single-host.html
--- /dev/null
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# Vagrantfile API/syntax version. Don"t touch unless you know what you"re doing!
+VAGRANTFILE_API_VERSION = "2".freeze
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ config.ssh.insert_key = false
+ config.ssh.forward_x11 = true
+
+ config.vm.define "arvados" do |arv|
+ arv.vm.box = "bento/debian-10"
+ arv.vm.hostname = "vagrant.local"
+ # CPU/RAM
+ config.vm.provider :virtualbox do |v|
+ v.memory = 2048
+ v.cpus = 2
+ end
+
+ # Networking
+ arv.vm.network "forwarded_port", guest: 8443, host: 8443
+ arv.vm.network "forwarded_port", guest: 25100, host: 25100
+ arv.vm.network "forwarded_port", guest: 9002, host: 9002
+ arv.vm.network "forwarded_port", guest: 9000, host: 9000
+ arv.vm.network "forwarded_port", guest: 8900, host: 8900
+ arv.vm.network "forwarded_port", guest: 8002, host: 8002
+ arv.vm.network "forwarded_port", guest: 8001, host: 8001
+ arv.vm.network "forwarded_port", guest: 8000, host: 8000
+ arv.vm.network "forwarded_port", guest: 3001, host: 3001
+ arv.vm.provision "shell",
+ path: "provision.sh",
+ args: [
+ # "--debug",
+ "--test",
+ "--vagrant",
+ "--ssl-port=8443"
+ ].join(" ")
+ end
+end
--- /dev/null
+#!/bin/bash
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# If you want to test arvados in a single host, you can run this script, which
+# will install it using salt masterless
+# This script is run by the Vagrant file when you run it with
+#
+# vagrant up
+
+##########################################################
+# This section are the basic parameters to configure the installation
+
+# The 5 letters name you want to give your cluster
+CLUSTER="arva2"
+DOMAIN="arv.local"
+
+INITIAL_USER="admin"
+
+# If not specified, the initial user email will be composed as
+# INITIAL_USER@CLUSTER.DOMAIN
+INITIAL_USER_EMAIL="${INITIAL_USER}@${CLUSTER}.${DOMAIN}"
+INITIAL_USER_PASSWORD="password"
+
+# The example config you want to use. Currently, only "single_host" is
+# available
+CONFIG_DIR="single_host"
+
+# Which release of Arvados repo you want to use
+RELEASE="production"
+# Which version of Arvados you want to install. Defaults to 'latest'
+# in the desired repo
+VERSION="latest"
+
+# Host SSL port where you want to point your browser to access Arvados
+# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
+# You can point it to another port if desired
+# In Vagrant, make sure it matches what you set in the Vagrantfile
+# HOST_SSL_PORT=443
+
+# This is a arvados-formula setting.
+# If branch is set, the script will switch to it before running salt
+# Usually not needed, only used for testing
+# BRANCH="master"
+
+##########################################################
+# Usually there's no need to modify things below this line
+
+# Formulas versions
+ARVADOS_TAG="v1.1.4"
+POSTGRES_TAG="v0.41.3"
+NGINX_TAG="v2.4.0"
+DOCKER_TAG="v1.0.0"
+LOCALE_TAG="v0.3.4"
+
+set -o pipefail
+
+# capture the directory that the script is running from
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+usage() {
+ echo >&2
+ echo >&2 "Usage: ${0} [-h] [-h]"
+ echo >&2
+ echo >&2 "${0} options:"
+ echo >&2 " -d, --debug Run salt installation in debug mode"
+ echo >&2 " -p <N>, --ssl-port <N> SSL port to use for the web applications"
+ echo >&2 " -t, --test Test installation running a CWL workflow"
+ echo >&2 " -h, --help Display this help and exit"
+ echo >&2 " -v, --vagrant Run in vagrant and use the /vagrant shared dir"
+ echo >&2
+}
+
+arguments() {
+ # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+ TEMP=$(getopt -o dhp:tv \
+ --long debug,help,ssl-port:,test,vagrant \
+ -n "${0}" -- "${@}")
+
+ if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+ # Note the quotes around `$TEMP': they are essential!
+ eval set -- "$TEMP"
+
+ while [ ${#} -ge 1 ]; do
+ case ${1} in
+ -d | --debug)
+ LOG_LEVEL="debug"
+ shift
+ ;;
+ -t | --test)
+ TEST="yes"
+ shift
+ ;;
+ -v | --vagrant)
+ VAGRANT="yes"
+ shift
+ ;;
+ -p | --ssl-port)
+ HOST_SSL_PORT=${2}
+ shift 2
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ usage
+ exit 1
+ ;;
+ esac
+ done
+}
+
+LOG_LEVEL="info"
+HOST_SSL_PORT=443
+TESTS_DIR="tests"
+
+arguments ${@}
+
+# Salt's dir
+## states
+S_DIR="/srv/salt"
+## formulas
+F_DIR="/srv/formulas"
+##pillars
+P_DIR="/srv/pillars"
+
+apt-get update
+apt-get install -y curl git jq
+
+dpkg -l |grep salt-minion
+if [ ${?} -eq 0 ]; then
+ echo "Salt already installed"
+else
+ curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+ sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+ /bin/systemctl disable salt-minion.service
+fi
+
+# Set salt to masterless mode
+cat > /etc/salt/minion << EOFSM
+file_client: local
+file_roots:
+ base:
+ - ${S_DIR}
+ - ${F_DIR}/*
+ - ${F_DIR}/*/test/salt/states/examples
+
+pillar_roots:
+ base:
+ - ${P_DIR}
+EOFSM
+
+mkdir -p ${S_DIR}
+mkdir -p ${F_DIR}
+mkdir -p ${P_DIR}
+
+# States
+cat > ${S_DIR}/top.sls << EOFTSLS
+base:
+ '*':
+ - single_host.host_entries
+ - single_host.snakeoil_certs
+ - locale
+ - nginx.passenger
+ - postgres
+ - docker
+ - arvados
+EOFTSLS
+
+# Pillars
+cat > ${P_DIR}/top.sls << EOFPSLS
+base:
+ '*':
+ - arvados
+ - docker
+ - locale
+ - nginx_api_configuration
+ - nginx_controller_configuration
+ - nginx_keepproxy_configuration
+ - nginx_keepweb_configuration
+ - nginx_passenger
+ - nginx_websocket_configuration
+ - nginx_webshell_configuration
+ - nginx_workbench2_configuration
+ - nginx_workbench_configuration
+ - postgresql
+EOFPSLS
+
+# Get the formula and dependencies
+cd ${F_DIR} || exit 1
+git clone --branch "${ARVADOS_TAG}" https://github.com/saltstack-formulas/arvados-formula.git
+git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
+git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
+git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
+git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
+
+if [ "x${BRANCH}" != "x" ]; then
+ cd ${F_DIR}/arvados-formula || exit 1
+ git checkout -t origin/"${BRANCH}"
+ cd -
+fi
+
+if [ "x${VAGRANT}" = "xyes" ]; then
+ SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+ TESTS_DIR="/vagrant/${TESTS_DIR}"
+else
+ SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
+ TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
+fi
+
+# Replace cluster and domain name in the example pillars and test files
+for f in "${SOURCE_PILLARS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
+ s/__RELEASE__/${RELEASE}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
+ s/__INITIAL_USER__/${INITIAL_USER}/g;
+ s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
+ s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
+ s/__VERSION__/${VERSION}/g" \
+ "${f}" > "${P_DIR}"/$(basename "${f}")
+done
+
+mkdir -p /tmp/cluster_tests
+# Replace cluster and domain name in the example pillars and test files
+for f in "${TESTS_DIR}"/*; do
+ sed "s/__CLUSTER__/${CLUSTER}/g;
+ s/__DOMAIN__/${DOMAIN}/g;
+ s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+ s/__INITIAL_USER__/${INITIAL_USER}/g;
+ s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
+ s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
+ ${f} > /tmp/cluster_tests/$(basename ${f})
+done
+chmod 755 /tmp/cluster_tests/run-test.sh
+
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ -e /root/.psqlrc ]; then
+ if ! ( grep 'pset pager off' /root/.psqlrc ); then
+ RESTORE_PSQL="yes"
+ cp /root/.psqlrc /root/.psqlrc.provision.backup
+ fi
+else
+ DELETE_PSQL="yes"
+fi
+
+echo '\pset pager off' >> /root/.psqlrc
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
+# Now run the install
+salt-call --local state.apply -l ${LOG_LEVEL}
+
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ "x${DELETE_PSQL}" = "xyes" ]; then
+ echo "Removing .psql file"
+ rm /root/.psqlrc
+fi
+
+if [ "x${RESTORE_PSQL}" = "xyes" ]; then
+ echo "Restoring .psql file"
+ mv -v /root/.psqlrc.provision.backup /root/.psqlrc
+fi
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
+# Leave a copy of the Arvados CA so the user can copy it where it's required
+echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
+# If running in a vagrant VM, also add default user to docker group
+if [ "x${VAGRANT}" = "xyes" ]; then
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant
+
+ echo "Adding the vagrant user to the docker group"
+ usermod -a -G docker vagrant
+else
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}
+fi
+
+# Test that the installation finished correctly
+if [ "x${TEST}" = "xyes" ]; then
+ cd /tmp/cluster_tests
+ ./run-test.sh
+fi
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The variables commented out are the default values that the formula uses.
+# The uncommented values are REQUIRED values. If you don't set them, running
+# this formula will fail.
+arvados:
+ ### GENERAL CONFIG
+ version: '__VERSION__'
+ ## It makes little sense to disable this flag, but you can, if you want :)
+ # use_upstream_repo: true
+
+ ## Repo URL is built with grains values. If desired, it can be completely
+ ## overwritten with the pillar parameter 'repo_url'
+ # repo:
+ # humanname: Arvados Official Repository
+
+ release: __RELEASE__
+
+ ## IMPORTANT!!!!!
+ ## api, workbench and shell require some gems, so you need to make sure ruby
+ ## and deps are installed in order to install and compile the gems.
+ ## We default to `false` in these two variables as it's expected you already
+ ## manage OS packages with some other tool and you don't want us messing up
+ ## with your setup.
+ ruby:
+ ## We set these to `true` here for testing purposes.
+ ## They both default to `false`.
+ manage_ruby: true
+ manage_gems_deps: true
+ # pkg: ruby
+ # gems_deps:
+ # - curl
+ # - g++
+ # - gcc
+ # - git
+ # - libcurl4
+ # - libcurl4-gnutls-dev
+ # - libpq-dev
+ # - libxml2
+ # - libxml2-dev
+ # - make
+ # - python3-dev
+ # - ruby-dev
+ # - zlib1g-dev
+
+ # config:
+ # file: /etc/arvados/config.yml
+ # user: root
+ ## IMPORTANT!!!!!
+ ## If you're intalling any of the rails apps (api, workbench), the group
+ ## should be set to that of the web server, usually `www-data`
+ # group: root
+ # mode: 640
+
+ ### ARVADOS CLUSTER CONFIG
+ cluster:
+ name: __CLUSTER__
+ domain: __DOMAIN__
+
+ database:
+ # max concurrent connections per arvados server daemon
+ # connection_pool_max: 32
+ name: arvados
+ host: 127.0.0.1
+ password: changeme_arvados
+ user: arvados
+ encoding: en_US.utf8
+ client_encoding: UTF8
+
+ tls:
+ # certificate: ''
+ # key: ''
+ # required to test with arvados-snakeoil certs
+ insecure: true
+
+ ### TOKENS
+ tokens:
+ system_root: changemesystemroottoken
+ management: changememanagementtoken
+ rails_secret: changemerailssecrettoken
+ anonymous_user: changemeanonymoususertoken
+
+ ### KEYS
+ secrets:
+ blob_signing_key: changemeblobsigningkey
+ workbench_secret_key: changemeworkbenchsecretkey
+ dispatcher_access_key: changemedispatcheraccesskey
+ dispatcher_secret_key: changeme_dispatchersecretkey
+ keep_access_key: changemekeepaccesskey
+ keep_secret_key: changemekeepsecretkey
+
+ Login:
+ Test:
+ Enable: true
+ Users:
+ __INITIAL_USER__:
+ Email: __INITIAL_USER_EMAIL__
+ Password: __INITIAL_USER_PASSWORD__
+
+ ### VOLUMES
+ ## This should usually match all your `keepstore` instances
+ Volumes:
+ # the volume name will be composed with
+ # <cluster>-nyw5e-<volume>
+ __CLUSTER__-nyw5e-000000000000000:
+ AccessViaHosts:
+ http://keep0.__CLUSTER__.__DOMAIN__:25107:
+ ReadOnly: false
+ Replication: 2
+ Driver: Directory
+ DriverParameters:
+ Root: /tmp
+
+ Users:
+ NewUsersAreActive: true
+ AutoAdminFirstUser: true
+ AutoSetupNewUsers: true
+ AutoSetupNewUsersWithRepository: true
+
+ Services:
+ Controller:
+ ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://controller.internal:8003: {}
+ DispatchCloud:
+ InternalURLs:
+ http://__CLUSTER__.__DOMAIN__:9006: {}
+ Keepbalance:
+ InternalURLs:
+ http://__CLUSTER__.__DOMAIN__:9005: {}
+ Keepproxy:
+ ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://keep.internal:25100: {}
+ Keepstore:
+ InternalURLs:
+ http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
+ RailsAPI:
+ InternalURLs:
+ http://api.internal:8004: {}
+ WebDAV:
+ ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ InternalURLs:
+ http://collections.internal:9002: {}
+ WebDAVDownload:
+ ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ WebShell:
+ ExternalURL: https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ Websocket:
+ ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
+ InternalURLs:
+ http://ws.internal:8005: {}
+ Workbench1:
+ ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+ Workbench2:
+ ExternalURL: https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+docker:
+ pkg:
+ docker:
+ use_upstream: package
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+locale:
+ present:
+ - "en_US.UTF-8 UTF-8"
+ default:
+ # Note: On debian systems don't write the second 'UTF-8' here or you will
+ # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8
+ # Restart the minion after you corrected this!
+ name: 'en_US.UTF-8'
+ requires: 'en_US.UTF-8 UTF-8'
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ arvados_api:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: 'api.internal:8004'
+ - server_name: api
+ - root: /var/www/arvados-api/current/public
+ - index: index.html index.htm
+ - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+ - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log
+ - passenger_enabled: 'on'
+ - client_max_body_size: 128m
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ 'geo $external_client':
+ default: 1
+ '127.0.0.0/8': 0
+ upstream controller_upstream:
+ - server: 'controller.internal:8003 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_controller_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: __CLUSTER__.__DOMAIN__
+ - listen:
+ - 80 default
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_controller_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: __CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://controller_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_set_header: 'X-External-Client $external_client'
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
+ - client_max_body_size: 128m
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream keepproxy_upstream:
+ - server: 'keep.internal:25100 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_keepproxy_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_keepproxy_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: keep.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://keepproxy_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream collections_downloads_upstream:
+ - server: 'collections.internal:9002 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_collections_download_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ ### COLLECTIONS / DOWNLOAD
+ arvados_collections_download_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://collections_downloads_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_max_body_size: 0
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ install_from_phusionpassenger: true
+ lookup:
+ passenger_package: libnginx-mod-http-passenger
+ passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+
+ ### SERVER
+ server:
+ config:
+ include: 'modules-enabled/*.conf'
+ worker_processes: 4
+
+ ### SITES
+ servers:
+ managed:
+ # Remove default webserver
+ default:
+ enabled: false
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ upstream webshell_upstream:
+ - server: 'shell.internal:4200 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ arvados_webshell_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: webshell.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_webshell_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: webshell.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /shell.__CLUSTER__.__DOMAIN__:
+ - proxy_pass: 'http://webshell_upstream'
+ - proxy_read_timeout: 90
+ - proxy_connect_timeout: 90
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_ssl_session_reuse: 'off'
+
+ - "if ($request_method = 'OPTIONS')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+ - add_header: "'Access-Control-Max-Age' 1728000"
+ - add_header: "'Content-Type' 'text/plain charset=UTF-8'"
+ - add_header: "'Content-Length' 0"
+ - return: 204
+
+ - "if ($request_method = 'POST')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+ - "if ($request_method = 'GET')":
+ - add_header: "'Access-Control-Allow-Origin' '*'"
+ - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+ - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
+
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream websocket_upstream:
+ - server: 'ws.internal:8005 fail_timeout=10s'
+
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_websocket_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_websocket_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: ws.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://websocket_upstream'
+ - proxy_read_timeout: 600
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: 'Host $host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'Upgrade $http_upgrade'
+ - proxy_set_header: 'Connection "upgrade"'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - proxy_buffering: 'off'
+ - client_body_buffer_size: 64M
+ - client_max_body_size: 64M
+ - proxy_http_version: '1.1'
+ - proxy_request_buffering: 'off'
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench2_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench2_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench2.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - root: /var/www/arvados-workbench2/workbench2
+ - try_files: '$uri $uri/ /index.html'
+ - 'if (-f $document_root/maintenance.html)':
+ - return: 503
+ - location /config.json:
+ - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__"}' ~ "'" }}
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+ config:
+ group: www-data
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+
+ ### STREAMS
+ http:
+ upstream workbench_upstream:
+ - server: 'workbench.internal:9000 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### DEFAULT
+ arvados_workbench_default:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ arvados_workbench_ssl:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: workbench.__CLUSTER__.__DOMAIN__
+ - listen:
+ - __HOST_SSL_PORT__ http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://workbench_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - include: 'snippets/arvados-snakeoil.conf'
+ - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
+
+ arvados_workbench_upstream:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - listen: 'workbench.internal:9000'
+ - server_name: workbench
+ - root: /var/www/arvados-workbench/current/public
+ - index: index.html index.htm
+ - passenger_enabled: 'on'
+ # yamllint disable-line rule:line-length
+ - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+ - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### POSTGRESQL
+postgres:
+ use_upstream_repo: false
+ pkgs_extra:
+ - postgresql-contrib
+ postgresconf: |-
+ listen_addresses = '*' # listen on all interfaces
+ acls:
+ - ['local', 'all', 'postgres', 'peer']
+ - ['local', 'all', 'all', 'peer']
+ - ['host', 'all', 'all', '127.0.0.1/32', 'md5']
+ - ['host', 'all', 'all', '::1/128', 'md5']
+ - ['host', 'arvados', 'arvados', '127.0.0.1/32']
+ users:
+ arvados:
+ ensure: present
+ password: changeme_arvados
+
+ # tablespaces:
+ # arvados_tablespace:
+ # directory: /path/to/some/tbspace/arvados_tbsp
+ # owner: arvados
+
+ databases:
+ arvados:
+ owner: arvados
+ template: template0
+ lc_ctype: en_US.utf8
+ lc_collate: en_US.utf8
+ # tablespace: arvados_tablespace
+ schemas:
+ public:
+ owner: arvados
+ extensions:
+ pg_trgm:
+ if_not_exists: true
+ schema: public
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+inputfile:
+ class: File
+ path: test.txt
+hasher1_outputname: hasher1.md5sum.txt
+hasher2_outputname: hasher2.md5sum.txt
+hasher3_outputname: hasher3.md5sum.txt
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+
+inputs:
+ inputfile: File
+ hasher1_outputname: string
+ hasher2_outputname: string
+ hasher3_outputname: string
+
+outputs:
+ hasher_out:
+ type: File
+ outputSource: hasher3/hasher_out
+
+steps:
+ hasher1:
+ run: hasher.cwl
+ in:
+ inputfile: inputfile
+ outputname: hasher1_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher2:
+ run: hasher.cwl
+ in:
+ inputfile: hasher1/hasher_out
+ outputname: hasher2_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
+
+ hasher3:
+ run: hasher.cwl
+ in:
+ inputfile: hasher2/hasher_out
+ outputname: hasher3_outputname
+ out: [hasher_out]
+ hints:
+ ResourceRequirement:
+ coresMin: 1
+ arv:IntermediateOutput:
+ outputTTL: 3600
+ arv:ReuseRequirement:
+ enableReuse: false
--- /dev/null
+#!/usr/bin/env cwl-runner
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+
+baseCommand: md5sum
+inputs:
+ inputfile:
+ type: File
+ inputBinding:
+ position: 1
+ outputname:
+ type: string
+
+stdout: $(inputs.outputname)
+
+outputs:
+ hasher_out:
+ type: File
+ outputBinding:
+ glob: $(inputs.outputname)
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+export ARVADOS_API_TOKEN=changemesystemroottoken
+export ARVADOS_API_HOST=__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+export ARVADOS_API_HOST_INSECURE=true
+
+set -o pipefail
+
+# First, validate that the CA is installed and that we can query it with no errors.
+if ! curl -s -o /dev/null https://workbench.${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then
+ echo "The Arvados CA was not correctly installed. Although some components will work,"
+ echo "others won't. Please verify that the CA cert file was installed correctly and"
+ echo "retry running these tests."
+ exit 1
+fi
+
+# https://doc.arvados.org/v2.0/install/install-jobs-image.html
+echo "Creating Arvados Standard Docker Images project"
+uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)
+project_uuid=$(arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]')
+
+if [ "x${project_uuid}" = "x" ]; then
+ project_uuid=$(arv --format=uuid group create --group "{\"owner_uuid\": \"${uuid_prefix}-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}")
+
+ read -rd $'\000' newlink <<EOF; arv link create --link "${newlink}"
+{
+ "tail_uuid":"${uuid_prefix}-j7d0g-fffffffffffffff",
+ "head_uuid":"${project_uuid}",
+ "link_class":"permission",
+ "name":"can_read"
+}
+EOF
+fi
+
+echo "Arvados project uuid is '${project_uuid}'"
+
+echo "Uploading arvados/jobs' docker image to the project"
+VERSION="2.1.1"
+arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
+
+# Create the initial user
+echo "Creating initial user '__INITIAL_USER__'"
+user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')
+
+if [ "x${user_uuid}" = "x" ]; then
+ user_uuid=$(arv --format=uuid user create --user '{"email": "__INITIAL_USER_EMAIL__", "username": "__INITIAL_USER__"}')
+ echo "Setting up user '__INITIAL_USER__'"
+ arv user setup --uuid "${user_uuid}"
+fi
+
+echo "Activating user '__INITIAL_USER__'"
+arv user update --uuid "${user_uuid}" --user '{"is_active": true}'
+
+echo "Getting the user API TOKEN"
+user_api_token=$(arv api_client_authorization list --filters "[[\"owner_uuid\", \"=\", \"${user_uuid}\"],[\"kind\", \"==\", \"arvados#apiClientAuthorization\"]]" --limit=1 |jq -r .items[].api_token)
+
+if [ "x${user_api_token}" = "x" ]; then
+ user_api_token=$(arv api_client_authorization create --api-client-authorization "{\"owner_uuid\": \"${user_uuid}\"}" | jq -r .api_token)
+fi
+
+# Change to the user's token and run the workflow
+export ARVADOS_API_TOKEN="${user_api_token}"
+
+echo "Running test CWL workflow"
+cwl-runner hasher-workflow.cwl hasher-workflow-job.yml
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+test
if !u.IsActive || !u.IsAdmin {
return config, fmt.Errorf("current user (%s) is not an active admin user", u.UUID)
}
- config.SysUserUUID = u.UUID[:12] + "000000000000000"
+
+ var ac struct{ ClusterID string }
+ err = config.Client.RequestAndDecode(&ac, "GET", "arvados/v1/config", nil, nil)
+ if err != nil {
+ return config, fmt.Errorf("error getting the exported config: %s", err)
+ }
+ config.SysUserUUID = ac.ClusterID + "-tpzed-000000000000000"
// Set up remote groups' parent
if err = SetParentGroup(&config); err != nil {
"group_class": "role",
}
if e := CreateGroup(cfg, &newGroup, groupData); e != nil {
- err = fmt.Errorf("error creating group named %q: %s", groupName, err)
+ err = fmt.Errorf("error creating group named %q: %s", groupName, e)
return
}
// Update cached group data
users map[string]arvados.User
}
-func (s *TestSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
-}
-
-func (s *TestSuite) TearDownSuite(c *C) {
- arvadostest.StopAPI()
-}
-
func (s *TestSuite) SetUpTest(c *C) {
ac := arvados.NewClientFromEnv()
u, err := ac.CurrentUser()
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+include agpl-3.0.txt
+include arvados_version.py
\ No newline at end of file
--- /dev/null
+.. Copyright (C) The Arvados Authors. All rights reserved.
+..
+.. SPDX-License-Identifier: AGPL-3.0
+
+Summarize user activity from Arvados audit logs
--- /dev/null
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
--- /dev/null
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import argparse
+import sys
+
+import arvados
+import arvados.util
+import datetime
+import ciso8601
+
+def parse_arguments(arguments):
+ arg_parser = argparse.ArgumentParser()
+ arg_parser.add_argument('--days', type=int, required=True)
+ args = arg_parser.parse_args(arguments)
+ return args
+
+def getowner(arv, uuid, owners):
+ if uuid is None:
+ return None
+ if uuid[6:11] == "tpzed":
+ return uuid
+
+ if uuid not in owners:
+ try:
+ gp = arv.groups().get(uuid=uuid).execute()
+ owners[uuid] = gp["owner_uuid"]
+ except:
+ owners[uuid] = None
+
+ return getowner(arv, owners[uuid], owners)
+
+def getuserinfo(arv, uuid):
+ u = arv.users().get(uuid=uuid).execute()
+ prof = "\n".join(" %s: \"%s\"" % (k, v) for k, v in u["prefs"].get("profile", {}).items() if v)
+ if prof:
+ prof = "\n"+prof+"\n"
+ return "%s %s <%s> (%susers/%s)%s" % (u["first_name"], u["last_name"], u["email"],
+ arv.config()["Services"]["Workbench1"]["ExternalURL"],
+ uuid, prof)
+
+def getname(u):
+ return "\"%s\" (%s)" % (u["name"], u["uuid"])
+
+def main(arguments=None):
+ if arguments is None:
+ arguments = sys.argv[1:]
+
+ args = parse_arguments(arguments)
+
+ arv = arvados.api()
+
+ since = datetime.datetime.utcnow() - datetime.timedelta(days=args.days)
+
+ print("User activity on %s between %s and %s\n" % (arv.config()["ClusterID"],
+ (datetime.datetime.now() - datetime.timedelta(days=args.days)).isoformat(sep=" ", timespec="minutes"),
+ datetime.datetime.now().isoformat(sep=" ", timespec="minutes")))
+
+ events = arvados.util.keyset_list_all(arv.logs().list, filters=[["created_at", ">=", since.isoformat()]])
+
+ users = {}
+ owners = {}
+
+ for e in events:
+ owner = getowner(arv, e["object_owner_uuid"], owners)
+ users.setdefault(owner, [])
+ event_at = ciso8601.parse_datetime(e["event_at"]).astimezone().isoformat(sep=" ", timespec="minutes")
+ # loguuid = e["uuid"]
+ loguuid = ""
+
+ if e["event_type"] == "create" and e["object_uuid"][6:11] == "tpzed":
+ users.setdefault(e["object_uuid"], [])
+ users[e["object_uuid"]].append("%s User account created" % event_at)
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "tpzed":
+ pass
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "xvhdp":
+ if e["properties"]["new_attributes"]["requesting_container_uuid"] is None:
+ users[owner].append("%s Ran container %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "xvhdp":
+ pass
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Created project %s" % (event_at, getname(e["properties"]["new_attributes"])))
+
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Deleted project %s" % (event_at, getname(e["properties"]["old_attributes"])))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "j7d0g":
+ users[owner].append("%s Updated project %s" % (event_at, getname(e["properties"]["new_attributes"])))
+
+ elif e["event_type"] in ("create", "update") and e["object_uuid"][6:11] == "gj3su":
+ since_last = None
+ if len(users[owner]) > 0 and users[owner][-1].endswith("activity"):
+ sp = users[owner][-1].split(" ")
+ start = sp[0]+" "+sp[1]
+ since_last = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(sp[3]+" "+sp[4])
+ span = ciso8601.parse_datetime(event_at) - ciso8601.parse_datetime(start)
+
+ if since_last is not None and since_last < datetime.timedelta(minutes=61):
+ users[owner][-1] = "%s to %s (%02d:%02d) Account activity" % (start, event_at, span.days*24 + int(span.seconds/3600), int((span.seconds % 3600)/60))
+ else:
+ users[owner].append("%s to %s (0:00) Account activity" % (event_at, event_at))
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "o0j2j":
+ if e["properties"]["new_attributes"]["link_class"] == "tag":
+ users[owner].append("%s Tagged %s" % (event_at, e["properties"]["new_attributes"]["head_uuid"]))
+ elif e["properties"]["new_attributes"]["link_class"] == "permission":
+ users[owner].append("%s Shared %s with %s" % (event_at, e["properties"]["new_attributes"]["tail_uuid"], e["properties"]["new_attributes"]["head_uuid"]))
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "o0j2j":
+ if e["properties"]["old_attributes"]["link_class"] == "tag":
+ users[owner].append("%s Untagged %s" % (event_at, e["properties"]["old_attributes"]["head_uuid"]))
+ elif e["properties"]["old_attributes"]["link_class"] == "permission":
+ users[owner].append("%s Unshared %s with %s" % (event_at, e["properties"]["old_attributes"]["tail_uuid"], e["properties"]["old_attributes"]["head_uuid"]))
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+
+ elif e["event_type"] == "create" and e["object_uuid"][6:11] == "4zz18":
+ if e["properties"]["new_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
+ pass
+ else:
+ users[owner].append("%s Created collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+
+ elif e["event_type"] == "update" and e["object_uuid"][6:11] == "4zz18":
+ users[owner].append("%s Updated collection %s %s" % (event_at, getname(e["properties"]["new_attributes"]), loguuid))
+
+ elif e["event_type"] == "delete" and e["object_uuid"][6:11] == "4zz18":
+ if e["properties"]["old_attributes"]["properties"].get("type") in ("log", "output", "intermediate"):
+ pass
+ else:
+ users[owner].append("%s Deleted collection %s %s" % (event_at, getname(e["properties"]["old_attributes"]), loguuid))
+
+ else:
+ users[owner].append("%s %s %s %s" % (e["event_type"], e["object_kind"], e["object_uuid"], loguuid))
+
+ for k,v in users.items():
+ if k is None or k.endswith("-tpzed-000000000000000"):
+ continue
+ print(getuserinfo(arv, k))
+ for ev in v:
+ print(" %s" % ev)
+ print("")
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import os
+import re
+import sys
+
+SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
+VERSION_PATHS = {
+ SETUP_DIR,
+ os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
+ os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
+ }
+
+def choose_version_from():
+ ts = {}
+ for path in VERSION_PATHS:
+ ts[subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%ct', path]).strip()] = path
+
+ sorted_ts = sorted(ts.items())
+ getver = sorted_ts[-1][1]
+ print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ return getver
+
+def git_version_at_commit():
+ curdir = choose_version_from()
+ myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
+ '--format=%H', curdir]).strip()
+ myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
+ return myversion
+
+def save_version(setup_dir, module, v):
+ v = v.replace("~dev", ".dev").replace("~rc", "rc")
+ with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
+ return fp.write("__version__ = '%s'\n" % v)
+
+def read_version(setup_dir, module):
+ with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
+ return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+
+def get_version(setup_dir, module):
+ env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+
+ if env_version:
+ save_version(setup_dir, module, env_version)
+ else:
+ try:
+ save_version(setup_dir, module, git_version_at_commit())
+ except (subprocess.CalledProcessError, OSError) as err:
+ print("ERROR: {0}".format(err), file=sys.stderr)
+ pass
+
+ return read_version(setup_dir, module)
--- /dev/null
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import arvados_user_activity.main
+
+arvados_user_activity.main.main()
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+case "$TARGET" in
+ debian* | ubuntu*)
+ fpm_depends+=(libcurl3-gnutls)
+ ;;
+esac
--- /dev/null
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+from __future__ import absolute_import
+import os
+import sys
+import re
+
+from setuptools import setup, find_packages
+
+SETUP_DIR = os.path.dirname(__file__) or '.'
+README = os.path.join(SETUP_DIR, 'README.rst')
+
+import arvados_version
+version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
+
+setup(name='arvados-user-activity',
+ version=version,
+ description='Summarize user activity from Arvados audit logs',
+ author='Arvados',
+ author_email='info@arvados.org',
+ url="https://arvados.org",
+ download_url="https://github.com/arvados/arvados.git",
+ license='GNU Affero General Public License, version 3.0',
+ packages=['arvados_user_activity'],
+ include_package_data=True,
+ entry_points={"console_scripts": ["arv-user-activity=arvados_user_activity.main:main"]},
+ data_files=[
+ ('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
+ ],
+ install_requires=[
+ 'arvados-python-client >= 2.2.0.dev20201118185221',
+ ],
+ zip_safe=True,
+)
projects / arvados.git / commitdiff