Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>
*.gz.report
*.ico
*.jpg
+*.svg
+*.odg
*.json
*LICENSE*.html
.licenseignore
services/keepproxy/pkg-extras/etc/default/keepproxy
*.tar
tools/crunchstat-summary/tests/crunchstat_error_messages.txt
+tools/crunchstat-summary/crunchstat_summary/synchronizer.js
+build/package-build-dockerfiles/debian9/D39DC0E3.asc
+build/package-test-dockerfiles/debian9/D39DC0E3.asc
+sdk/R/DESCRIPTION
+sdk/R/NAMESPACE
+sdk/R/.Rbuildignore
+sdk/R/ArvadosR.Rproj
+*.Rd
President and Fellows of Harvard College <*@harvard.edu>
Thomas Mooney <tmooney@genome.wustl.edu>
Chen Chen <aflyhorse@gmail.com>
+Veritas Genetics, Inc. <*@veritasgenetics.com>
class UsersController < ApplicationController
skip_around_filter :require_thread_api_token, only: :welcome
- skip_before_filter :check_user_agreements, only: [:welcome, :inactive]
- skip_before_filter :check_user_profile, only: [:welcome, :inactive, :profile]
+ skip_before_filter :check_user_agreements, only: [:welcome, :inactive, :link_account, :merge]
+ skip_before_filter :check_user_profile, only: [:welcome, :inactive, :profile, :link_account, :merge]
skip_before_filter :find_object_by_uuid, only: [:welcome, :activity, :storage]
before_filter :ensure_current_user_is_admin, only: [:sudo, :unsetup, :setup]
RequestShellAccessReporter.send_request(current_user, params).deliver
end
+ def merge
+ User.merge params[:new_user_token], params[:direction]
+ redirect_to "/"
+ end
+
protected
def find_current_links user
arvados_api_client.unpack_api_response(res)
end
+ def self.merge new_user_token, direction
+ # Merge user accounts.
+ #
+ # If the direction is "in", the current user is merged into the
+ # user represented by new_user_token
+ #
+ # If the direction is "out", the user represented by new_user_token
+ # is merged into the current user.
+
+ if direction == "in"
+ user_a = new_user_token
+ user_b = Thread.current[:arvados_api_token]
+ new_group_name = "Migrated from #{Thread.current[:user].email} (#{Thread.current[:user].uuid})"
+ elsif direction == "out"
+ user_a = Thread.current[:arvados_api_token]
+ user_b = new_user_token
+ res = arvados_api_client.api self, '/current', nil, {:arvados_api_token => user_b}, false
+ user_b_info = arvados_api_client.unpack_api_response(res)
+ new_group_name = "Migrated from #{user_b_info.email} (#{user_b_info.uuid})"
+ else
+ raise "Invalid merge direction, expected 'in' or 'out'"
+ end
+
+ # Create a project owned by user_a to accept everything owned by user_b
+ res = arvados_api_client.api Group, nil, {:group => {
+ :name => new_group_name,
+ :group_class => "project"},
+ :ensure_unique_name => true},
+ {:arvados_api_token => user_a}, false
+ target = arvados_api_client.unpack_api_response(res)
+
+ # The merge API merges the "current" user (user_b) into the user
+ # represented by "new_user_token" (user_a).
+ # After merging, the user_b redirects to user_a.
+ res = arvados_api_client.api self, '/merge', {:new_user_token => user_a,
+ :new_owner_uuid => target[:uuid],
+ :redirect_to_new_user => true},
+ {:arvados_api_token => user_b}, false
+ arvados_api_client.unpack_api_response(res)
+ end
+
def self.system
@@arvados_system_user ||= begin
res = arvados_api_client.api self, '/system'
<%= link_to ssh_keys_user_path(current_user), role: 'menu-item' do %>
<i class="fa fa-lg fa-key fa-fw"></i> SSH keys
<% end %>
- </li>
+</li>
+ <li role="menuitem"><a href="/users/link_account" role="menuitem"><i class="fa fa-lg fa-link fa-fw"></i> Link account </a></li>
<% if Rails.configuration.user_profile_form_fields %>
<li role="menuitem"><a href="/users/<%=current_user.uuid%>/profile" role="menuitem"><i class="fa fa-lg fa-user fa-fw"></i> Manage profile</a></li>
<% end %>
+<!-- Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: AGPL-3.0 -->
+
<div data-mount-mithril="TestComponent"></div>
<%= link_to 'Retry', (params[:return_to] || '/'), class: 'btn btn-primary' %>
</p>
+
+ <p>
+ Already have an account with a different login? <a href="/users/link_account">Link this login to your existing account.</a>
+ </p>
+
</div>
</div>
</div>
--- /dev/null
+<%# Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: AGPL-3.0 %>
+
+<%= javascript_tag do %>
+ function update_visibility() {
+ if (sessionStorage.getItem('link_account_api_token') &&
+ sessionStorage.getItem('link_account_uuid') != '<%= Thread.current[:user].uuid %>')
+ {
+ $("#ready-to-link").css({"display": "inherit"});
+ $("#need-login").css({"display": "none"});
+
+ <% if params[:direction] == "in" %>
+ var user_a = "<b>"+sessionStorage.getItem('link_account_email')+"</b> ("+sessionStorage.getItem('link_account_username')+", "+sessionStorage.getItem('link_account_uuid')+")";
+ var user_b = "<b><%= Thread.current[:user].email %></b> (<%= Thread.current[:user].username%>, <%= Thread.current[:user].uuid%>)";
+ var user_a_is_active = (sessionStorage.getItem('link_account_is_active') == "true");
+ var user_a_is_admin = (sessionStorage.getItem('link_account_is_admin') == "true");
+ var user_b_is_admin = <%=if Thread.current[:user].is_admin then "true" else "false" end %>;
+ <% else %>
+ var user_a = "<b><%= Thread.current[:user].email %></b> (<%= Thread.current[:user].username%>, <%= Thread.current[:user].uuid%>)";
+ var user_b = "<b>"+sessionStorage.getItem('link_account_email')+"</b> ("+sessionStorage.getItem('link_account_username')+", "+sessionStorage.getItem('link_account_uuid')+")";
+ var user_a_is_active = <%= Thread.current[:user].is_active %>;
+ var user_a_is_admin = <%=if Thread.current[:user].is_admin then "true" else "false" end %>;
+ var user_b_is_admin = (sessionStorage.getItem('link_account_is_admin') == "true");
+ <% end %>
+
+ $("#new-user-token-input").val(sessionStorage.getItem('link_account_api_token'));
+
+ if (!user_a_is_active) {
+ $("#will-link-to").html("<p>Cannot link "+user_b+" to inactive account "+user_a+".</p>");
+ $("#link-account-submit").prop("disabled", true);
+ } else if (user_b_is_admin && !user_a_is_admin) {
+ $("#will-link-to").html("<p>Cannot link admin account "+user_b+" to non-admin account "+user_a+".</p>");
+ $("#link-account-submit").prop("disabled", true);
+ } else {
+ $("#will-link-to").html("<p>Clicking 'Link accounts' will link "+user_b+" created on <%=Thread.current[:user].created_at%> to "+
+ user_a+" created at <b>"+sessionStorage.getItem('link_account_created_at')+"</b>.</p>"+
+ "<p>After linking, logging in as "+user_b+" will log you into the same account as "+user_a+
+ ".</p> <p>Any objects owned by "+user_b+" will be transferred to "+user_a+".</p>");
+ }
+ } else {
+ $("#ready-to-link").css({"display": "none"});
+ $("#need-login").css({"display": "inherit"});
+ }
+
+ sessionStorage.removeItem('link_account_api_token');
+ sessionStorage.removeItem('link_account_uuid');
+ sessionStorage.removeItem('link_account_email');
+ sessionStorage.removeItem('link_account_username');
+ sessionStorage.removeItem('link_account_created_at');
+ sessionStorage.removeItem('link_account_is_active');
+ sessionStorage.removeItem('link_account_is_admin');
+ };
+
+ $(window).on("load", function() {
+ update_visibility();
+ });
+
+ function do_login(dir) {
+ sessionStorage.setItem('link_account_api_token', '<%= Thread.current[:arvados_api_token] %>');
+ sessionStorage.setItem('link_account_email', '<%= Thread.current[:user].email %>');
+ sessionStorage.setItem('link_account_username', '<%= Thread.current[:user].username %>');
+ sessionStorage.setItem('link_account_uuid', '<%= Thread.current[:user].uuid %>');
+ sessionStorage.setItem('link_account_created_at', '<%= Thread.current[:user].created_at %>');
+ sessionStorage.setItem('link_account_is_active', <%= if Thread.current[:user].is_active then "true" else "false" end %>);
+ sessionStorage.setItem('link_account_is_admin', <%= if Thread.current[:user].is_admin then "true" else "false" end %>);
+ window.location.replace('<%=arvados_api_client.arvados_logout_url(return_to: arvados_api_client.arvados_login_url(return_to: "#{strip_token_from_path(request.url)}?direction="))%>'+dir);
+ }
+
+ $(document).on("click", "#link-account-in", function(e) { do_login("in"); });
+ $(document).on("click", "#link-account-out", function(e) { do_login("out"); });
+
+ $(document).on("click", "#cancel-link-accounts", function() {
+ window.location.replace('/users/link_account?api_token='+$("#new-user-token-input").val());
+ });
+<% end %>
+
+<div id="need-login" style="display: none">
+
+ <p>You are currently logged in as <b><%= Thread.current[:user].email %></b> (<%= Thread.current[:user].username%>, <%= Thread.current[:user].uuid %>) created at <b><%= Thread.current[:user].created_at%></b></p>
+
+<p>You can link Arvados accounts. After linking, either login will take you to the same account.</p>
+
+ <p>
+ <% if Thread.current[:user].is_active %>
+ <button class="btn btn-primary" id="link-account-in" style="margin-right: 1em">
+ <i class="fa fa-fw fa-sign-in"></i> Add another login to this account
+ </button>
+ <% end %>
+ <button class="btn btn-primary" id="link-account-out" style="margin-right: 1em">
+ <i class="fa fa-fw fa-sign-in"></i> Use this login to access another account
+ </button>
+
+</p>
+</div>
+
+<div id="ready-to-link" style="display: none">
+
+ <div id="will-link-to"></div>
+
+ <%= button_tag "Cancel", class: "btn btn-cancel pull-left", id: "cancel-link-accounts", style: "margin-right: 1em" %>
+
+ <%= form_tag do |f| %>
+ <input type="hidden" id="new-user-token-input" name="new_user_token" value="" />
+ <input type="hidden" id="new-user-token-input" name="direction" value="<%=params[:direction]%>" />
+ <%= button_tag class: "btn btn-primary", id: "link-account-submit" do %>
+ <i class="fa fa-fw fa-link"></i> Link accounts
+ <% end %>
+<% end %>
+
+</div>
+</div>
get 'virtual_machines', :on => :member
get 'repositories', :on => :member
get 'ssh_keys', :on => :member
+ get 'link_account', :on => :collection
+ post 'link_account', :on => :collection, :action => :merge
end
get '/current_token' => 'users#current_token'
get "/add_ssh_key_popup" => 'users#add_ssh_key_popup', :as => :add_ssh_key_popup
get 'tab_counts', on: :member
get 'public', on: :collection
end
-
+
resources :search do
get 'choose', :on => :collection
end
match '/_health/ping', to: 'healthcheck#ping', via: [:get]
get '/tests/mithril', to: 'tests#mithril'
-
+
get '/status', to: 'status#status'
-
+
# Send unroutable requests to an arbitrary controller
# (ends up at ApplicationController#render_not_found)
match '*a', to: 'links#render_not_found', via: [:get, :post]
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'integration_helper'
+require 'webrick'
+
+class LinkAccountTest < ActionDispatch::IntegrationTest
+ setup do
+ need_javascript
+ end
+
+ def start_sso_stub token
+ port = available_port('sso_stub')
+
+ s = WEBrick::HTTPServer.new(
+ :Port => port,
+ :BindAddress => 'localhost',
+ :Logger => WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG),
+ :AccessLog => [nil,nil]
+ )
+
+ s.mount_proc("/login"){|req, res|
+ res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, req.query["return_to"] + "&api_token=#{token}")
+ s.shutdown
+ }
+
+ s.mount_proc("/logout"){|req, res|
+ res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, req.query["return_to"])
+ }
+
+ Thread.new do
+ s.start
+ end
+
+ "http://localhost:#{port}/"
+ end
+
+ test "Add another login to this account" do
+ visit page_with_token('active_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['project_viewer_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ find("a", text: "Link account").click
+ find("button", text: "Add another login to this account").click
+
+ find("#notifications-menu").click
+ assert_text "project-viewer@arvados.local"
+
+ find("button", text: "Link accounts").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+ end
+
+ test "Use this login to access another account" do
+ visit page_with_token('project_viewer_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['active_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "project-viewer@arvados.local"
+
+ find("a", text: "Link account").click
+ find("button", text: "Use this login to access another account").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ find("button", text: "Link accounts").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+ end
+
+ test "Link login of inactive user to this account" do
+ visit page_with_token('active_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['inactive_uninvited_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ find("a", text: "Link account").click
+ find("button", text: "Add another login to this account").click
+
+ find("#notifications-menu").click
+ assert_text "inactive-uninvited-user@arvados.local"
+
+ find("button", text: "Link accounts").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+ end
+
+ test "Cannot link to inactive user" do
+ visit page_with_token('active_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['inactive_uninvited_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ find("a", text: "Link account").click
+ find("button", text: "Use this login to access another account").click
+
+ find("#notifications-menu").click
+ assert_text "inactive-uninvited-user@arvados.local"
+
+ assert_text "Cannot link active-user@arvados.local"
+
+ assert find("#link-account-submit")['disabled']
+
+ find("button", text: "Cancel").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+ end
+
+ test "Inactive user can link to active account" do
+ visit page_with_token('inactive_uninvited_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['active_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "inactive-uninvited-user@arvados.local"
+
+ assert_text "Already have an account with a different login?"
+
+ find("a", text: "Link this login to your existing account").click
+
+ assert_no_text "Add another login to this account"
+
+ find("button", text: "Use this login to access another account").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ find("button", text: "Link accounts").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+ end
+
+ test "Admin cannot link to non-admin" do
+ visit page_with_token('admin_trustedclient')
+ stub = start_sso_stub(api_fixture('api_client_authorizations')['active_trustedclient']['api_token'])
+ Rails.configuration.arvados_login_base = stub + "login"
+
+ find("#notifications-menu").click
+ assert_text "admin@arvados.local"
+
+ find("a", text: "Link account").click
+ find("button", text: "Use this login to access another account").click
+
+ find("#notifications-menu").click
+ assert_text "active-user@arvados.local"
+
+ assert_text "Cannot link admin account admin@arvados.local"
+
+ assert find("#link-account-submit")['disabled']
+
+ find("button", text: "Cancel").click
+
+ find("#notifications-menu").click
+ assert_text "admin@arvados.local"
+ end
+
+end
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
debian8,debian9,ubuntu1404,ubuntu1604,centos7|lockfile|0.12.2|2|python|all|--epoch 1
-debian8,debian9,ubuntu1404,ubuntu1604,centos7|subprocess32|3.5.0rc1|2|python|all
+debian8,debian9,ubuntu1404,ubuntu1604,centos7|subprocess32|3.5.1|2|python|all
all|ruamel.yaml|0.14.12|2|python|amd64|--python-setup-py-arguments --single-version-externally-managed
-all|cwltest|1.0.20180416154033|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32'
+all|cwltest|1.0.20180518074130|4|python|all|--depends 'python-futures >= 3.0.5' --depends 'python-subprocess32 >= 3.5.0'
all|junit-xml|1.8|3|python|all
all|rdflib-jsonld|0.4.0|2|python|all
all|futures|3.0.5|2|python|all
| */nodemanager/doc/*.cfg \
| */nodemanager/tests/fake*.cfg.template \
| */nginx.conf \
- | build/build.list)
+ | build/build.list | *.R)
fixer=fixer
cc="#"
;;
wantBYSAmd="[comment]: # (Copyright © The Arvados Authors. All rights reserved.)
[comment]: # ()
[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0)"
- found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*Arvados' || true)
+ found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*All rights reserved.' || true)
case ${fnm} in
Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job)
want=${wantGPL}
fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0rc1" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" "${iterargs[@]}"
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --depends "${PYTHON2_PKG_PREFIX}-subprocess32 >= 3.5.0" --depends "${PYTHON2_PKG_PREFIX}-pathlib2" --depends "${PYTHON2_PKG_PREFIX}-scandir" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
- user/cwl/cwl-extensions.html.textile.liquid
- user/topics/arv-docker.html.textile.liquid
- Reference:
+ - user/topics/link-accounts.html.textile.liquid
- user/reference/cookbook.html.textile.liquid
- Arvados License:
- user/copying/copying.html.textile.liquid
- admin/upgrading.html.textile.liquid
- install/cheat_sheet.html.textile.liquid
- user/topics/arvados-sync-groups.html.textile.liquid
+ - admin/migrating-providers.html.textile.liquid
- admin/merge-remote-account.html.textile.liquid
- install/migrate-docker19.html.textile.liquid
installguide:
--- /dev/null
+---
+layout: default
+navsection: admin
+title: "Migrating account providers"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page describes how to enable users to use more than one provider to log into the same Arvados account. This can be used to migrate account providers, for example, from LDAP to Google. In order to do this, users must be able to log into both the "old" and "new" providers.
+
+h2. Configure multiple providers in SSO
+
+In @application.yml@ for the SSO server, enable both @google_oauth2@ and @ldap@ providers:
+
+<pre>
+production:
+ google_oauth2_client_id: abcd
+ google_oauth2_client_secret: abcd
+
+ use_ldap:
+ title: Example LDAP
+ host: ldap.example.com
+ port: 636
+ method: ssl
+ base: "ou=Users, dc=example, dc=com"
+ uid: uid
+ username: uid
+</pre>
+
+Restart the SSO server after changing the configuration.
+
+h2. Link accounts
+
+Instruct users to go through the process of "linking accounts":{{site.baseurl}}/user/topics/link-accounts.html
+
+After linking accounts, users can use the new provider to access their existing Arvados account.
+
+Once all users have migrated, the old account provider can be removed from the SSO configuration.
+/* Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0 */
+
img.full-width {
width: 100%
}
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Linking alternate login accounts"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+This page describes how to link additional login accounts to the same Arvados account. This can be used to migrate login accounts, for example, from one Google account to another. It can also be used to migrate login providers, for example from LDAP to Google. In order to do this, you must be able to log into both the "old" and "new" accounts.
+
+h2. Link accounts
+
+Follow this process to link the "new" login to the "old" login.
+
+# Log in using the "old" account
+# Under the users menu, choose *Link account*
+# On the link accounts page, press the button *Add another login to this account*
+# Follow login instructions from the login provider (eg Google)
+# You will be returned to the *Link accounts* confirmation page.
+# Press the *Link account* button to confirm.
+# After the accounts are linked, you will be returned to the dashboard.
+# Both the "old" and "new" logins will now log in to the same Arvados account.
+
+h2. Link accounts (alternate flow)
+
+You can also link accounts starting with logging into the "new" account first.
+
+# Log in using the "new" account
+# Under the users menu, choose *Link account* (if the user is inactive, there will be a link on the inactive user page)
+# On the link accounts page, press the button *Use this login to access another account*
+# Follow login instructions from the login provider (eg Google)
+# You will be returned to the *Link accounts* confirmation page.
+# Press the *Link account* button to confirm.
+# After the accounts are linked, you will be returned to the dashboard.
+# Both the "old" and "new" logins will now log in to the same Arvados account.
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
#' users.get
#'
#' users.get is a method defined in Arvados class.
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("./R/util.R")
#' ArvadosFile
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("./R/Subcollection.R")
source("./R/ArvadosFile.R")
source("./R/RESTService.R")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("./R/Subcollection.R")
source("./R/ArvadosFile.R")
source("./R/util.R")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
HttpParser <- R6::R6Class(
"HttrParser",
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("./R/util.R")
HttpRequest <- R6::R6Class(
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
RESTService <- R6::R6Class(
"RESTService",
headers <- list(Authorization = paste("OAuth2", self$token))
- serverResponse <- self$http$execute("GET", discoveryDocumentURL, headers,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("GET", discoveryDocumentURL, headers,
+ retryTimes = self$numRetries)
discoveryDocument <- self$httpParser$parseJSONResponse(serverResponse)
private$webDavHostName <- discoveryDocument$keepWebServiceUrl
uuid, "/", relativePath);
headers <- list(Authorization = paste("OAuth2", self$token))
- serverResponse <- self$http$execute("DELETE", fileURL, headers,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("DELETE", fileURL, headers,
+ retryTimes = self$numRetries)
if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
stop(paste("Server code:", serverResponse$status_code))
headers <- list("Authorization" = paste("OAuth2", self$token),
"Destination" = toURL)
- serverResponse <- self$http$execute("MOVE", fromURL, headers,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("MOVE", fromURL, headers,
+ retryTimes = self$numRetries)
if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
stop(paste("Server code:", serverResponse$status_code))
headers <- list("Authorization" = paste("OAuth2", self$token))
- response <- self$http$execute("PROPFIND", collectionURL, headers,
- retryTimes = self$numRetries)
+ response <- self$http$exec("PROPFIND", collectionURL, headers,
+ retryTimes = self$numRetries)
if(all(response == ""))
stop("Response is empty, request may be misconfigured")
headers <- list("Authorization" = paste("OAuth2", self$token))
- response <- self$http$execute("PROPFIND", subcollectionURL, headers,
- retryTimes = self$numRetries)
+ response <- self$http$exec("PROPFIND", subcollectionURL, headers,
+ retryTimes = self$numRetries)
if(all(response == ""))
stop("Response is empty, request may be misconfigured")
if(!(contentType %in% self$httpParser$validContentTypes))
stop("Invalid contentType. Please use text or raw.")
- serverResponse <- self$http$execute("GET", fileURL, headers,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("GET", fileURL, headers,
+ retryTimes = self$numRetries)
if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
stop(paste("Server code:", serverResponse$status_code))
"Content-Type" = contentType)
body <- content
- serverResponse <- self$http$execute("PUT", fileURL, headers, body,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("PUT", fileURL, headers, body,
+ retryTimes = self$numRetries)
if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
stop(paste("Server code:", serverResponse$status_code))
"Content-Type" = contentType)
body <- NULL
- serverResponse <- self$http$execute("PUT", fileURL, headers, body,
- retryTimes = self$numRetries)
+ serverResponse <- self$http$exec("PUT", fileURL, headers, body,
+ retryTimes = self$numRetries)
if(serverResponse$status_code < 200 || serverResponse$status_code >= 300)
stop(paste("Server code:", serverResponse$status_code))
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("./R/util.R")
#' Subcollection
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
getAPIDocument <- function(){
url <- "https://4xphq.arvadosapi.com/discovery/v1/apis/arvados/v1/rest"
serverResponse <- httr::RETRY("GET", url = url)
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
#' listAll
#'
#' List all resources even if the number of items is greater than maximum API limit.
+[comment]: # (Copyright © The Arvados Authors. All rights reserved.)
+[comment]: # ()
+[comment]: # (SPDX-License-Identifier: CC-BY-SA-3.0)
+
## R SDK for Arvados
This SDK focuses on providing support for accessing Arvados projects, collections, and the files within collections.
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
#Run script with $Rscript createDoc.R input.Rmd output.html
require(knitr) # required for knitting from rmd to md
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
options(repos=structure(c(CRAN="http://cran.wustl.edu/")))
if (!requireNamespace("devtools")) {
install.packages("devtools")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
results <- devtools::test()
any_error <- any(as.data.frame(results)$error)
if (any_error) {
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
library(testthat)
library(ArvadosR)
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
FakeArvados <- R6::R6Class(
"FakeArvados",
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
FakeHttpParser <- R6::R6Class(
"FakeHttrParser",
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
FakeHttpRequest <- R6::R6Class(
"FakeHttpRequest",
self$serverMaxElementsPerRequest <- 5
},
- execute = function(verb, url, headers = NULL, body = NULL, query = NULL,
- limit = NULL, offset = NULL, retryTimes = 0)
+ exec = function(verb, url, headers = NULL, body = NULL, query = NULL,
+ limit = NULL, offset = NULL, retryTimes = 0)
{
private$validateURL(url)
private$validateHeaders(headers)
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
FakeRESTService <- R6::R6Class(
"FakeRESTService",
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("fakes/FakeRESTService.R")
context("ArvadosFile")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("fakes/FakeRESTService.R")
context("Collection")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
context("CollectionTree")
test_that("constructor creates file tree from character array properly", {
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
context("Http Parser")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
context("Http Request")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("fakes/FakeArvados.R")
source("fakes/FakeHttpRequest.R")
source("fakes/FakeHttpParser.R")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
source("fakes/FakeRESTService.R")
context("Subcollection")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
context("Utility function")
test_that("listAll always returns all resource items from server", {
# Workaround for #13365
builderargs = kwargs.copy()
builderargs["toplevel"] = True
+ builderargs["tmp_outdir_prefix"] = ""
builder = self._init_job(joborder, **builderargs)
joborder = builder.job
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to build.
install_requires=[
- 'cwltool==1.0.20180508202931',
+ 'cwltool==1.0.20180522135731',
'schema-salad==2.7.20180501211602',
'typing==3.5.3.0',
'ruamel.yaml >=0.13.11, <0.15',
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: ExpressionTool
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
dir:
class: Directory
location: samples
\ No newline at end of file
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
{
"cwlVersion": "v1.0",
"arguments": [
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
i:
class: File
location: keep:f225e6259bdd63bc7240599648dde9f1+97/hg19.fa
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
class: Workflow
cwlVersion: v1.0
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
class: CommandLineTool
cwlVersion: v1.0
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
toplevel_input:
class: File
location: keep:4d8a70b1e63b2aad6984e40e338e2373+69/hello.txt
\ No newline at end of file
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
pw: blorp
basedir="", make_fs_access=make_fs_access, loader=document_loader,
makeTool=runner.arv_make_tool, metadata=metadata)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
+ it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
it.next().run()
it.next().run()
basedir="", make_fs_access=make_fs_access, loader=document_loader,
makeTool=runner.arv_make_tool, metadata=metadata)
arvtool.formatgraph = None
- it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access)
+ it = arvtool.job({}, mock.MagicMock(), basedir="", make_fs_access=make_fs_access, tmp_outdir_prefix="")
it.next().run()
it.next().run()
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
inputs:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
inputs: []
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import sys
import os
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
requirements:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: CommandLineTool
$namespaces:
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
cwlVersion: v1.0
class: Workflow
$namespaces:
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package arvados
import (
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package health
import (
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package health
import (
@must_be_writable
@synchronized
@retry_method
- def save(self, merge=True, num_retries=None):
+ def save(self, storage_classes=None, merge=True, num_retries=None):
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
the API server. If you want to save a manifest to Keep only, see
`save_new()`.
+ :storage_classes:
+ Specify desirable storage classes to be used when writing data to Keep.
+
:merge:
Update and merge remote changes before saving. Otherwise, any
remote changes will be ignored and overwritten.
Retry count on API calls (if None, use the collection default)
"""
+ if storage_classes and type(storage_classes) is not list:
+ raise errors.ArgumentError("storage_classes must be list type.")
+
if not self.committed():
if not self._has_collection_uuid():
raise AssertionError("Collection manifest_locator is not a collection uuid. Use save_new() for new collections.")
self.update()
text = self.manifest_text(strip=False)
+ body={'manifest_text': text}
+ if storage_classes:
+ body["storage_classes_desired"] = storage_classes
+
self._remember_api_response(self._my_api().collections().update(
uuid=self._manifest_locator,
- body={'manifest_text': text}
+ body=body
).execute(
num_retries=num_retries))
self._manifest_text = self._api_response["manifest_text"]
self._portable_data_hash = self._api_response["portable_data_hash"]
self.set_committed(True)
+ elif storage_classes:
+ self._remember_api_response(self._my_api().collections().update(
+ uuid=self._manifest_locator,
+ body={"storage_classes_desired": storage_classes}
+ ).execute(
+ num_retries=num_retries))
return self._manifest_text
def save_new(self, name=None,
create_collection_record=True,
owner_uuid=None,
+ storage_classes=None,
ensure_unique_name=False,
num_retries=None):
"""Save collection to a new collection record.
the user, or project uuid that will own this collection.
If None, defaults to the current user.
+ :storage_classes:
+ Specify desirable storage classes to be used when writing data to Keep.
+
:ensure_unique_name:
If True, ask the API server to rename the collection
if it conflicts with a collection with the same name and owner. If
"replication_desired": self.replication_desired}
if owner_uuid:
body["owner_uuid"] = owner_uuid
+ if storage_classes:
+ if type(storage_classes) is not list:
+ raise errors.ArgumentError("storage_classes must be list type.")
+ body["storage_classes_desired"] = storage_classes
self._remember_api_response(self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries))
text = self._api_response["manifest_text"]
block. Default is to use the server-provided default (if any) or 2.
""")
+upload_opts.add_argument('--storage-classes', help="""
+Specify comma separated list of storage classes to be used when saving data to Keep.
+""")
+
upload_opts.add_argument('--threads', type=int, metavar='N', default=None,
help="""
Set the number of upload threads to be used. Take into account that
def __init__(self, paths, resume=True, use_cache=True, reporter=None,
name=None, owner_uuid=None, api_client=None,
ensure_unique_name=False, num_retries=None,
- put_threads=None, replication_desired=None,
- filename=None, update_time=60.0, update_collection=None,
+ put_threads=None, replication_desired=None, filename=None,
+ update_time=60.0, update_collection=None, storage_classes=None,
logger=logging.getLogger('arvados.arv_put'), dry_run=False,
follow_links=True, exclude_paths=[], exclude_names=None):
self.paths = paths
self.replication_desired = replication_desired
self.put_threads = put_threads
self.filename = filename
+ self.storage_classes = storage_classes
self._api_client = api_client
self._state_lock = threading.Lock()
self._state = None # Previous run state (file list & manifest)
else:
# The file already exist on remote collection, skip it.
pass
- self._remote_collection.save(num_retries=self.num_retries)
+ self._remote_collection.save(storage_classes=self.storage_classes,
+ num_retries=self.num_retries)
else:
+ if self.storage_classes is None:
+ self.storage_classes = ['default']
self._local_collection.save_new(
name=self.name, owner_uuid=self.owner_uuid,
+ storage_classes=self.storage_classes,
ensure_unique_name=self.ensure_unique_name,
num_retries=self.num_retries)
else:
reporter = None
+ # Split storage-classes argument
+ storage_classes = None
+ if args.storage_classes:
+ storage_classes = args.storage_classes.strip().split(',')
+ if len(storage_classes) > 1:
+ logger.error("Multiple storage classes are not supported currently.")
+ sys.exit(1)
+
+
# Setup exclude regex from all the --exclude arguments provided
name_patterns = []
exclude_paths = []
owner_uuid = project_uuid,
ensure_unique_name = True,
update_collection = args.update_collection,
+ storage_classes=storage_classes,
logger=logger,
dry_run=args.dry_run,
follow_links=args.follow_links,
self.call_main_with_args,
['--project-uuid', self.Z_UUID, '--stream'])
+ def test_error_when_multiple_storage_classes_specified(self):
+ self.assertRaises(SystemExit,
+ self.call_main_with_args,
+ ['--storage-classes', 'hot,cold'])
+
def test_error_when_excluding_absolute_path(self):
tmpdir = self.make_tmpdir()
self.assertRaises(SystemExit,
'--project-uuid', self.PROJECT_UUID])
self.assertEqual(link_name, collection['name'])
+ def test_put_collection_with_storage_classes_specified(self):
+ collection = self.run_and_find_collection("", ['--storage-classes', 'hot'])
+
+ self.assertEqual(len(collection['storage_classes_desired']), 1)
+ self.assertEqual(collection['storage_classes_desired'][0], 'hot')
+
+ def test_put_collection_without_storage_classes_specified(self):
+ collection = self.run_and_find_collection("")
+
+ self.assertEqual(len(collection['storage_classes_desired']), 1)
+ self.assertEqual(collection['storage_classes_desired'][0], 'default')
+
def test_exclude_filename_pattern(self):
tmpdir = self.make_tmpdir()
tmpsubdir = os.path.join(tmpdir, 'subdir')
def test_create_and_save(self):
c = self.create_count_txt()
- c.save()
+ c.save(storage_classes=['archive'])
+
self.assertRegex(
c.manifest_text(),
r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
+ self.assertEqual(c.api_response()["storage_classes_desired"], ['archive'])
+
def test_create_and_save_new(self):
c = self.create_count_txt()
- c.save_new()
+ c.save_new(storage_classes=['archive'])
+
self.assertRegex(
c.manifest_text(),
r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count\.txt$",)
+ self.assertEqual(c.api_response()["storage_classes_desired"], ['archive'])
+
+ def test_update_storage_classes_desired_if_collection_is_commited(self):
+ c = self.create_count_txt()
+ c.save(storage_classes=['hot'])
+ c.save(storage_classes=['cold'])
+
+ self.assertEqual(c.api_response()["storage_classes_desired"], ['cold'])
def test_create_diff_apply(self):
c1 = self.create_count_txt()
show
end
- # Updates use row locking to resolve races between multiple
- # dispatchers trying to lock the same container.
def update
- @object.with_lock do
+ # container updates can trigger container request lookups, which
+ # can deadlock if we don't lock the container_requests table
+ # first.
+ @object.transaction do
+ ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+ @object.reload
super
end
end
end
while (uuid = user.redirect_to_user_uuid)
- user = User.where(uuid: uuid).first
+ user = User.unscoped.where(uuid: uuid).first
if !user
raise Exception.new("identity_url #{omniauth['info']['identity_url']} redirects to nonexistent uuid #{uuid}")
end
# (because state might have changed while acquiring the lock).
check_lock_fail
transaction do
- begin
- reload(lock: 'FOR UPDATE NOWAIT')
- rescue
- raise LockFailedError.new("cannot lock: other transaction in progress")
- end
+ # Locking involves assigning auth_uuid, which involves looking
+ # up container requests, so we must lock both tables in the
+ # proper order to avoid deadlock.
+ ActiveRecord::Base.connection.execute('LOCK container_requests, containers IN EXCLUSIVE MODE')
+ reload
check_lock_fail
update_attributes!(state: Locked)
end
end
if self.is_active_changed?
if self.is_active != self.is_active_was
- logger.warn "User #{current_user.uuid} tried to change is_active from #{self.is_admin_was} to #{self.is_admin} for #{self.uuid}"
+ logger.warn "User #{current_user.uuid} tried to change is_active from #{self.is_active_was} to #{self.is_active} for #{self.uuid}"
self.is_active = self.is_active_was
end
end
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
require 'migrate_yaml_to_json'
class YamlToJson < ActiveRecord::Migration
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
require './db/migrate/20161213172944_full_text_search_indexes'
class JsonCollectionProperties < ActiveRecord::Migration
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class AddIndexToContainers < ActiveRecord::Migration
def up
ActiveRecord::Base.connection.execute("CREATE INDEX index_containers_on_modified_at_uuid ON containers USING btree (modified_at desc, uuid asc)")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class FixTrashFlagFollow < ActiveRecord::Migration
def change
ActiveRecord::Base.connection.execute("DROP MATERIALIZED VIEW materialized_permission_view")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class AddGinIndexToCollectionProperties < ActiveRecord::Migration
def up
ActiveRecord::Base.connection.execute("CREATE INDEX collection_index_on_properties ON collections USING gin (properties);")
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class AddSecretMountsToContainers < ActiveRecord::Migration
def change
add_column :container_requests, :secret_mounts, :jsonb, default: {}
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class ChangeContainerPriorityBigint < ActiveRecord::Migration
def change
change_column :containers, :priority, :integer, limit: 8
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class AddRedirectToUserUuidToUsers < ActiveRecord::Migration
def up
add_column :users, :redirect_to_user_uuid, :string
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
class AddContainerAuthUuidIndex < ActiveRecord::Migration
def change
add_index :containers, :auth_uuid
api_token: 62mhllc0otp78v08e3rpa3nsmf8q8ogk47f7u5z4erp5gpj9al
expires_at: 2038-01-01 00:00:00
+inactive_uninvited_trustedclient:
+ uuid: zzzzz-gj3su-228z32aux8dg2s1
+ api_client: trusted_workbench
+ user: inactive_uninvited
+ api_token: 7s29oj2hzmcmpq80hx9cta0rl5wuf3xfd6r7disusaptz7h9m0
+ expires_at: 2038-01-01 00:00:00
+
inactive_but_signed_user_agreement:
uuid: zzzzz-gj3su-247z32aux8dg2s1
api_client: untrusted
organization: example.com
role: IT
getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
+redirects_to_active:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-1au3is3g3chtthd
+ email: redirects-to-active-user@arvados.local
+ first_name: Active2
+ last_name: User2
+ identity_url: https://redirects-to-active-user.openid.local
+ is_active: true
+ is_admin: false
+ username: redirect_active
+ redirect_to_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
+ getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
+double_redirects_to_active:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zzzzz-tpzed-oiusowoxoz0pk3p
+ email: double-redirects-to-active-user@arvados.local
+ first_name: Active3
+ last_name: User3
+ identity_url: https://double-redirects-to-active-user.openid.local
+ is_active: true
+ is_admin: false
+ username: double_redirect_active
+ redirect_to_user_uuid: zzzzz-tpzed-1au3is3g3chtthd
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
+ getting_started_shown: 2015-03-26 12:34:56.789000000 Z
'https://wb.example.com'
end
- def mock_auth_with(email: nil, username: nil)
+ def mock_auth_with(email: nil, username: nil, identity_url: nil)
mock = {
'provider' => 'josh_id',
'uid' => 'https://edward.example.com',
}
mock['info']['email'] = email unless email.nil?
mock['info']['username'] = username unless username.nil?
+ mock['info']['identity_url'] = identity_url unless identity_url.nil?
post('/auth/josh_id/callback',
{return_to: client_url},
{'omniauth.auth' => mock})
assert_equal 'foo', u.username
end
+ test 'existing user login' do
+ mock_auth_with(identity_url: "https://active-user.openid.local")
+ u = assigns(:user)
+ assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ end
+
+ test 'user redirect_to_user_uuid' do
+ mock_auth_with(identity_url: "https://redirects-to-active-user.openid.local")
+ u = assigns(:user)
+ assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ end
+
+ test 'user double redirect_to_user_uuid' do
+ mock_auth_with(identity_url: "https://double-redirects-to-active-user.openid.local")
+ u = assigns(:user)
+ assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ end
+
test 'create new user during omniauth callback' do
mock_auth_with(email: 'edward@example.com')
assert_equal(0, @response.redirect_url.index(client_url),
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
import (
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
import (
"flag"
"fmt"
"log"
+ "net/http"
"os"
"os/signal"
"syscall"
// more memory, but can reduce store-and-forward latency when
// fetching pages)
CollectionBuffers int
+
+ // Timeout for outgoing http request/response cycle.
+ RequestTimeout arvados.Duration
}
// RunOptions controls runtime behavior. The flags/options that belong
log.Fatal(config.DumpAndExit(cfg))
}
+ to := time.Duration(cfg.RequestTimeout)
+ if to == 0 {
+ to = 30 * time.Minute
+ }
+ arvados.DefaultSecureClient.Timeout = to
+ arvados.InsecureHTTPClient.Timeout = to
+ http.DefaultClient.Timeout = to
+
log.Printf("keep-balance %s started", version)
if *debugFlag {
- disk
RunPeriod: 600s
CollectionBatchSize: 100000
-CollectionBuffers: 1000`)
+CollectionBuffers: 1000
+RequestTimeout: 30m`)
func usage() {
fmt.Fprintf(os.Stderr, `
while the current page is still being processed. If this is zero
or omitted, pages are processed serially.
+ RequestTimeout is the maximum time keep-balance will spend on a
+ single HTTP request (getting a page of collections, getting the
+ block index from a keepstore server, or sending a trash or pull
+ list to a keepstore server). Defaults to 30 minutes.
+
Limitations:
keep-balance does not attempt to discover whether committed pull
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
import "golang.org/x/net/webdav"
#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
# system one time tasks
PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
PATH=/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/X11R6/bin
#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
exec 2>&1
PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
exec 2>&1
PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
+// Copyright (c) 2009 Dan Vanderkam. All rights reserved.
+//
+// SPDX-License-Identifier: MIT
+
/**
* Synchronize zooming and/or selections between a set of dygraphs.
*
* You may also set `range: false` if you wish to only sync the x-axis.
* The `range` option has no effect unless `zoom` is true (the default).
*
- * SPDX-License-Identifier: MIT
* Original source: https://github.com/danvk/dygraphs/blob/master/src/extras/synchronizer.js
* at commit b55a71d768d2f8de62877c32b3aec9e9975ac389
*
projects / arvados.git / commitdiff