From: Tom Clegg Date: Mon, 10 Jul 2017 20:19:19 +0000 (-0400) Subject: Merge branch '11917-dont-clear-cache' X-Git-Tag: 1.1.0~147 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/d084ca24b06c598271844d2ba4c8c40f251c0999?hp=e8d11e963248fada51bdb1ecc59446adccff440f Merge branch '11917-dont-clear-cache' refs #11917 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/.licenseignore b/.licenseignore index 1d0356b51e..51980b16c2 100644 --- a/.licenseignore +++ b/.licenseignore @@ -1,7 +1,9 @@ *agpl-3.0.html *agpl-3.0.txt +apache-2.0.txt apps/workbench/app/assets/javascripts/list.js apps/workbench/public/webshell/* +AUTHORS */bootstrap.css */bootstrap.js *bootstrap-theme.css diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000000..ad72aea0aa --- /dev/null +++ b/AUTHORS @@ -0,0 +1,18 @@ +# Names should be added to this file with this pattern: +# +# For individuals: +# Name +# +# For organizations: +# Organization +# +# See python fnmatch module documentation for more information. + +Curoverse, Inc. <*@curoverse.com> +Adam Savitzky +Colin Nolan +David +Guillermo Carrasco +Joshua Randall +President and Fellows of Harvard College <*@harvard.edu> +Thomas Mooney diff --git a/COPYING b/COPYING index 2cba2ad20a..61c31397a0 100644 --- a/COPYING +++ b/COPYING @@ -1,15 +1,19 @@ -Server-side components of Arvados contained in the apps/ and services/ -directories, including the API Server, Workbench, and Crunch, are licensed -under the GNU Affero General Public License version 3 (see agpl-3.0.txt). +Unless indicated otherwise in the header of the file, the files in this +repository are distributed under one of three different licenses: AGPL-3.0, +Apache-2.0 or CC-BY-SA-3.0. -The files and directories under the build/, lib/ and tools/ directories are -licensed under the GNU Affero General Public License version 3 (see -agpl-3.0.txt). +Individual files contain an SPDX tag that indicates the license for the file. +These are the three tags in use: -The Arvados client Software Development Kits contained in the sdk/ directory, -example scripts in the crunch_scripts/ directory, the files and directories -under backports/ and docker/, and code samples in the Aravados documentation -are licensed under the Apache License, Version 2.0 (see LICENSE-2.0.txt). + SPDX-License-Identifier: AGPL-3.0 + SPDX-License-Identifier: Apache-2.0 + SPDX-License-Identifier: CC-BY-SA-3.0 -The Arvados Documentation located in the doc/ directory is licensed under the -Creative Commons Attribution-Share Alike 3.0 United States (see by-sa-3.0.txt). +This enables machine processing of license information based on the SPDX +License Identifiers that are available here: http://spdx.org/licenses/ + +The full license text for each license is available in this directory: + + AGPL-3.0: agpl-3.0.txt + Apache-2.0: apache-2.0.txt + CC-BY-SA-3.0: cc-by-sa-3.0.txt diff --git a/LICENSE-2.0.txt b/apache-2.0.txt similarity index 100% rename from LICENSE-2.0.txt rename to apache-2.0.txt diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb index 99065947b8..f8fcf5108f 100644 --- a/apps/workbench/app/controllers/collections_controller.rb +++ b/apps/workbench/app/controllers/collections_controller.rb @@ -115,6 +115,10 @@ class CollectionsController < ApplicationController end def show_file_links + if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url + # show_file will redirect to keep-web's directory listing + return show_file + end Thread.current[:reader_tokens] = [params[:reader_token]] return if false.equal?(find_object_by_uuid) render layout: false @@ -283,7 +287,12 @@ class CollectionsController < ApplicationController helper_method :download_link def download_link - collections_url + "/download/#{@object.uuid}/#{@search_sharing.first.api_token}/" + token = @search_sharing.first.api_token + if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url + keep_web_url(@object.uuid, nil, {path_token: token}) + else + collections_url + "/download/#{@object.uuid}/#{token}/" + end end def share @@ -444,7 +453,7 @@ class CollectionsController < ApplicationController uri.path += 't=' + opts[:path_token] + '/' end uri.path += '_/' - uri.path += URI.escape(file) + uri.path += URI.escape(file) if file query = Hash[URI.decode_www_form(uri.query || '')] { query_token: 'api_token', diff --git a/apps/workbench/app/controllers/repositories_controller.rb b/apps/workbench/app/controllers/repositories_controller.rb index 2a39a94018..5ca6f22b02 100644 --- a/apps/workbench/app/controllers/repositories_controller.rb +++ b/apps/workbench/app/controllers/repositories_controller.rb @@ -6,7 +6,7 @@ class RepositoriesController < ApplicationController before_filter :set_share_links, if: -> { defined? @object } def index_pane_list - %w(recent help) + %w(repositories help) end def show_pane_list @@ -36,4 +36,72 @@ class RepositoriesController < ApplicationController def show_commit @commit = params[:commit] end + + def all_repos + limit = params[:limit].andand.to_i || 100 + offset = params[:offset].andand.to_i || 0 + @filters = params[:filters] || [] + + if @filters.any? + owner_filter = @filters.select do |attr, op, val| + (attr == 'owner_uuid') + end + end + + if !owner_filter.andand.any? + filters = @filters + [["owner_uuid", "=", current_user.uuid]] + my_repos = Repository.all.order("name ASC").limit(limit).offset(offset).filter(filters).results + else # done fetching all owned repositories + my_repos = [] + end + + if !owner_filter.andand.any? # if this is next page request, the first page was still fetching "own" repos + @filters = @filters.reject do |attr, op, val| + (attr == 'owner_uuid') or + (attr == 'name') or + (attr == 'uuid') + end + end + + filters = @filters + [["owner_uuid", "!=", current_user.uuid]] + other_repos = Repository.all.order("name ASC").limit(limit).offset(offset).filter(filters).results + + @objects = (my_repos + other_repos).first(limit) + end + + def find_objects_for_index + return if !params[:partial] + + all_repos + + if @objects.any? + @next_page_filters = next_page_filters('>=') + @next_page_href = url_for(partial: :repositories_rows, + filters: @next_page_filters.to_json) + else + @next_page_href = nil + end + end + + def next_page_href with_params={} + @next_page_href + end + + def next_page_filters nextpage_operator + next_page_filters = @filters.reject do |attr, op, val| + (attr == 'owner_uuid') or + (attr == 'name' and op == nextpage_operator) or + (attr == 'uuid' and op == 'not in') + end + + if @objects.any? + last_obj = @objects.last + next_page_filters += [['name', nextpage_operator, last_obj.name]] + next_page_filters += [['uuid', 'not in', [last_obj.uuid]]] + # if not-owned, it means we are done with owned repos and fetching other repos + next_page_filters += [['owner_uuid', '!=', last_obj.uuid]] if last_obj.owner_uuid != current_user.uuid + end + + next_page_filters + end end diff --git a/apps/workbench/app/controllers/users_controller.rb b/apps/workbench/app/controllers/users_controller.rb index 4a40f03120..2e3ced69a5 100644 --- a/apps/workbench/app/controllers/users_controller.rb +++ b/apps/workbench/app/controllers/users_controller.rb @@ -260,28 +260,6 @@ class UsersController < ApplicationController end end - def repositories - # all repositories accessible by current user - all_repositories = Hash[Repository.all.order('name asc').collect {|repo| [repo.uuid, repo]}] - - @my_repositories = [] # we want them ordered as owned and the rest - @repo_writable = {} - - # owned repos - all_repositories.each do |_, repo| - if repo.owner_uuid == current_user.uuid - @repo_writable[repo.uuid] = 'can_write' - @my_repositories << repo - end - end - - # rest of the repos - handled = @my_repositories.map(&:uuid) - all_repositories.each do |_, repo| - @my_repositories << repo if !repo.uuid.in?(handled) - end - end - def virtual_machines @my_vm_logins = {} Link.where(tail_uuid: @object.uuid, diff --git a/apps/workbench/app/views/collections/_show_tags.html.erb b/apps/workbench/app/views/collections/_show_tags.html.erb index fd734c82dc..afab5266e9 100644 --- a/apps/workbench/app/views/collections/_show_tags.html.erb +++ b/apps/workbench/app/views/collections/_show_tags.html.erb @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0 %>
<% if object.editable? %>

- Edit + Edit

<% end %> diff --git a/apps/workbench/app/views/layouts/body.html.erb b/apps/workbench/app/views/layouts/body.html.erb index 8073615c31..3315027b3d 100644 --- a/apps/workbench/app/views/layouts/body.html.erb +++ b/apps/workbench/app/views/layouts/body.html.erb @@ -66,11 +66,7 @@ SPDX-License-Identifier: AGPL-3.0 %> Virtual machines <% end %> -
  • - <%= link_to repositories_user_path(current_user), role: 'menu-item' do %> - Repositories - <% end %> -
    • +
  • Repositories
  • Current token
  • <%= link_to ssh_keys_user_path(current_user), role: 'menu-item' do %> diff --git a/apps/workbench/app/views/users/_add_repository_modal.html.erb b/apps/workbench/app/views/repositories/_add_repository_modal.html.erb similarity index 100% rename from apps/workbench/app/views/users/_add_repository_modal.html.erb rename to apps/workbench/app/views/repositories/_add_repository_modal.html.erb diff --git a/apps/workbench/app/views/repositories/_show_help.html.erb b/apps/workbench/app/views/repositories/_show_help.html.erb index 7980738b55..5904fb29db 100644 --- a/apps/workbench/app/views/repositories/_show_help.html.erb +++ b/apps/workbench/app/views/repositories/_show_help.html.erb @@ -2,7 +2,13 @@ SPDX-License-Identifier: AGPL-3.0 %> -<% if (example = @objects.select(&:push_url).first) %> +<% + filters = @filters + [["owner_uuid", "=", current_user.uuid]] + example = Repository.all.order("name ASC").filter(filters).limit(1).results.first + example = Repository.all.order("name ASC").limit(1).results.first if !example +%> + +<% if example %>

    Sample git quick start: diff --git a/apps/workbench/app/views/repositories/_show_repositories.html.erb b/apps/workbench/app/views/repositories/_show_repositories.html.erb new file mode 100644 index 0000000000..871ba1d365 --- /dev/null +++ b/apps/workbench/app/views/repositories/_show_repositories.html.erb @@ -0,0 +1,46 @@ +<%# Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: AGPL-3.0 %> + +<%= render partial: "add_repository_modal" %> + +

    +
    +
    +

    + When you are using an Arvados virtual machine, you should clone the https:// URLs. This will authenticate automatically using your API token. +

    +

    + In order to clone git repositories using SSH, <%= link_to ssh_keys_user_path(current_user) do%> add an SSH key to your account<%end%> and clone the git@ URLs. +

    +
    +
    + <%= link_to raw(' Add new repository'), "#", + {class: 'btn btn-xs btn-primary', 'data-toggle' => "modal", + 'data-target' => '#add-repository-modal'} %> +
    +
    + +
    + + + + + + + + + + + + + + + + + + +
    Name URL
    +
    +
    diff --git a/apps/workbench/app/views/repositories/_show_repositories_rows.html.erb b/apps/workbench/app/views/repositories/_show_repositories_rows.html.erb new file mode 100644 index 0000000000..fe88608d78 --- /dev/null +++ b/apps/workbench/app/views/repositories/_show_repositories_rows.html.erb @@ -0,0 +1,23 @@ +<%# Copyright (C) The Arvados Authors. All rights reserved. + +SPDX-License-Identifier: AGPL-3.0 %> + +<% @objects.each do |repo| %> + + + <%= render :partial => "show_object_button", :locals => {object: repo, size: 'xs' } %> + + + <%= repo[:name] %> + + + <%= repo.http_fetch_url %>
    + <%= repo.editable? ? repo.push_url : repo.fetch_url %> + + + <% if repo.editable? %> + <%= render partial: 'delete_object_button', locals: {object: repo} %> + <% end %> + + +<% end %> diff --git a/apps/workbench/app/views/users/_repositories.html.erb b/apps/workbench/app/views/users/_repositories.html.erb deleted file mode 100644 index 057b6a2138..0000000000 --- a/apps/workbench/app/views/users/_repositories.html.erb +++ /dev/null @@ -1,71 +0,0 @@ -<%# Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: AGPL-3.0 %> - -
    -
    -
    - <%= link_to raw(' Add new repository'), "#", - {class: 'btn btn-xs btn-primary', 'data-toggle' => "modal", - 'data-target' => '#add-repository-modal'} %> -
    -

    - <%= link_to repositories_user_path(current_user) do%> - Repositories - <%end%> -

    -
    - -
    -

    - When you are using an Arvados virtual machine, you should clone the https:// URLs. This will authenticate automatically using your API token. -

    -

    - In order to clone git repositories using SSH, <%= link_to ssh_keys_user_path(current_user) do%> add an SSH key to your account<%end%> and clone the git@ URLs. -

    - - <% if !@my_repositories.any? %> - You do not seem to have access to any repositories. If you would like to request access, please contact your system admin. - <% else %> - - - - - - - - - - - - - - - - - <% @my_repositories.andand.each do |repo| %> - - - - - - - <% end %> - -
    Name URL
    - <%= render :partial => "show_object_button", :locals => {object: repo, size: 'xs' } %> - - <%= repo[:name] %> - - <%= repo.http_fetch_url %>
    - <%= @repo_writable[repo.uuid] ? repo.push_url : repo.fetch_url %> -     		- <% if repo.editable? %> - <%= link_to(repository_path(id: repo.uuid), method: :delete, class: 'btn btn-sm', data: {confirm: "Really delete '#{repo.name || repo.uuid}'?"}) do %> - - <% end %> - <% end %> -
    - <% end %> -
    -
    diff --git a/apps/workbench/app/views/users/repositories.html.erb b/apps/workbench/app/views/users/repositories.html.erb deleted file mode 100644 index 8a1a3b99d3..0000000000 --- a/apps/workbench/app/views/users/repositories.html.erb +++ /dev/null @@ -1,6 +0,0 @@ -<%# Copyright (C) The Arvados Authors. All rights reserved. - -SPDX-License-Identifier: AGPL-3.0 %> - -<%= render :partial => 'repositories' %> -<%= render partial: "add_repository_modal" %> diff --git a/apps/workbench/test/controllers/repositories_controller_test.rb b/apps/workbench/test/controllers/repositories_controller_test.rb index 561723da54..99e7285b3b 100644 --- a/apps/workbench/test/controllers/repositories_controller_test.rb +++ b/apps/workbench/test/controllers/repositories_controller_test.rb @@ -125,4 +125,20 @@ class RepositoriesControllerTest < ActionController::TestCase assert_select 'tr td', 'COPYING' end end + + test "get repositories lists linked as well as owned repositories" do + params = { + partial: :repositories_rows, + format: :json, + } + get :index, params, session_for(:active) + assert_response :success + repos = assigns(:objects) + assert repos + assert_not_empty repos, "my_repositories should not be empty" + repo_uuids = repos.map(&:uuid) + assert_includes repo_uuids, api_fixture('repositories')['repository2']['uuid'] # owned by active + assert_includes repo_uuids, api_fixture('repositories')['repository4']['uuid'] # shared with active + assert_includes repo_uuids, api_fixture('repositories')['arvados']['uuid'] # shared with all_users + end end diff --git a/apps/workbench/test/controllers/users_controller_test.rb b/apps/workbench/test/controllers/users_controller_test.rb index ee64969f76..ce9282ff77 100644 --- a/apps/workbench/test/controllers/users_controller_test.rb +++ b/apps/workbench/test/controllers/users_controller_test.rb @@ -35,29 +35,6 @@ class UsersControllerTest < ActionController::TestCase assert_match /\/users\/welcome/, @response.redirect_url end - test "show repositories with read, write, or manage permission" do - get :repositories, {id: api_fixture("users")['active']['uuid']}, session_for(:active) - assert_response :success - repos = assigns(:my_repositories) - assert repos - assert_not_empty repos, "my_repositories should not be empty" - editables = repos.collect { |r| !!assigns(:repo_writable)[r.uuid] } - assert_includes editables, true, "should have a writable repository" - assert_includes editables, false, "should have a readonly repository" - end - - test "show repositories lists linked as well as owned repositories" do - get :repositories, {id: api_fixture("users")['active']['uuid']}, session_for(:active) - assert_response :success - repos = assigns(:my_repositories) - assert repos - assert_not_empty repos, "my_repositories should not be empty" - repo_uuids = repos.map(&:uuid) - assert_includes repo_uuids, api_fixture('repositories')['repository2']['uuid'] # owned by active - assert_includes repo_uuids, api_fixture('repositories')['repository4']['uuid'] # shared with active - assert_includes repo_uuids, api_fixture('repositories')['arvados']['uuid'] # shared with all_users - end - test "request shell access" do user = api_fixture('users')['spectator'] diff --git a/apps/workbench/test/integration/application_layout_test.rb b/apps/workbench/test/integration/application_layout_test.rb index e777ebe784..7692d8e5dc 100644 --- a/apps/workbench/test/integration/application_layout_test.rb +++ b/apps/workbench/test/integration/application_layout_test.rb @@ -56,7 +56,7 @@ class ApplicationLayoutTest < ActionDispatch::IntegrationTest assert_selector "a[href=\"/projects/#{user['uuid']}\"]", text: 'Home project' assert_selector "a[href=\"/users/#{user['uuid']}/virtual_machines\"]", text: 'Virtual machines' - assert_selector "a[href=\"/users/#{user['uuid']}/repositories\"]", text: 'Repositories' + assert_selector "a[href=\"/repositories\"]", text: 'Repositories' assert_selector "a[href=\"/current_token\"]", text: 'Current token' assert_selector "a[href=\"/users/#{user['uuid']}/ssh_keys\"]", text: 'SSH keys' @@ -214,7 +214,7 @@ class ApplicationLayoutTest < ActionDispatch::IntegrationTest end [ - ['Repositories', nil, 's0uqq'], + ['Repositories', nil, 'active/crunchdispatchtest'], ['Virtual machines', nil, 'testvm.shell'], ['SSH keys', nil, 'public_key'], ['Links', nil, 'link_class'], diff --git a/apps/workbench/test/integration/collection_upload_test.rb b/apps/workbench/test/integration/collection_upload_test.rb index e54a5c2185..608cd521de 100644 --- a/apps/workbench/test/integration/collection_upload_test.rb +++ b/apps/workbench/test/integration/collection_upload_test.rb @@ -96,11 +96,11 @@ class CollectionUploadTest < ActionDispatch::IntegrationTest test "Report network error" do need_selenium "to make file uploads work" use_token :admin do - # Even if you somehow do port>2^16, surely nx.example.net won't + # Even if port 0 is a thing, surely nx.example.net won't # respond KeepService.where(service_type: 'proxy').first. update_attributes(service_host: 'nx.example.net', - service_port: 99999) + service_port: 0) end visit page_with_token 'active', sandbox_path diff --git a/apps/workbench/test/integration/collections_test.rb b/apps/workbench/test/integration/collections_test.rb index af8e1313f8..8619858dfe 100644 --- a/apps/workbench/test/integration/collections_test.rb +++ b/apps/workbench/test/integration/collections_test.rb @@ -426,8 +426,6 @@ class CollectionsTest < ActionDispatch::IntegrationTest end test "collection tags tab" do - need_selenium - visit page_with_token('active', '/collections/zzzzz-4zz18-bv31uwvy3neko21') click_link 'Tags' @@ -447,8 +445,6 @@ class CollectionsTest < ActionDispatch::IntegrationTest assert_selector 'a', text: 'Cancel' # add two tags - first('.edit-collection-tags').click - first('.glyphicon-plus').click first('.collection-tag-field-key').click first('.collection-tag-field-key').set('key 1') diff --git a/apps/workbench/test/integration/download_test.rb b/apps/workbench/test/integration/download_test.rb index 37faef9c3d..407458b62b 100644 --- a/apps/workbench/test/integration/download_test.rb +++ b/apps/workbench/test/integration/download_test.rb @@ -8,6 +8,8 @@ require 'helpers/download_helper' class DownloadTest < ActionDispatch::IntegrationTest include KeepWebConfig + @@wrote_test_data = false + setup do use_keep_web_config @@ -17,10 +19,13 @@ class DownloadTest < ActionDispatch::IntegrationTest # Keep data isn't populated by fixtures, so we have to write any # data we expect to read. - ['foo', 'w a z', "Hello world\n"].each do |data| - md5 = `echo -n #{data.shellescape} | arv-put --no-progress --raw -` - assert_match /^#{Digest::MD5.hexdigest(data)}/, md5 - assert $?.success?, $? + if !@@wrote_test_data + ['foo', 'w a z', "Hello world\n"].each do |data| + md5 = `echo -n #{data.shellescape} | arv-put --no-progress --raw -` + assert_match /^#{Digest::MD5.hexdigest(data)}/, md5 + assert $?.success?, $? + end + @@wrote_test_data = true end end @@ -29,7 +34,7 @@ class DownloadTest < ActionDispatch::IntegrationTest uuid_or_pdh = api_fixture('collections')['foo_file'][id_type] token = api_fixture('api_client_authorizations')['active_all_collections']['api_token'] visit "/collections/download/#{uuid_or_pdh}/#{token}/" - within "#collection_files" do + within 'ul' do click_link 'foo' end assert_no_selector 'a' diff --git a/apps/workbench/test/integration/user_settings_menu_test.rb b/apps/workbench/test/integration/user_settings_menu_test.rb index 75009f709c..6a0e46e26a 100644 --- a/apps/workbench/test/integration/user_settings_menu_test.rb +++ b/apps/workbench/test/integration/user_settings_menu_test.rb @@ -114,7 +114,7 @@ class UserSettingsMenuTest < ActionDispatch::IntegrationTest end test "verify repositories for active user" do - visit page_with_token('active',"/users/#{api_fixture('users')['active']['uuid']}/repositories") + visit page_with_token('active',"/repositories") repos = [[api_fixture('repositories')['foo'], true], [api_fixture('repositories')['repository3'], false], @@ -126,9 +126,9 @@ class UserSettingsMenuTest < ActionDispatch::IntegrationTest assert_text repo['name'] assert_selector 'a', text:'Show' if owned - assert_not_nil first('.fa-trash-o') + assert_not_nil first('.glyphicon-trash') else - assert_nil first('.fa-trash-o') + assert_nil first('.glyphicon-trash') end end end @@ -193,12 +193,12 @@ class UserSettingsMenuTest < ActionDispatch::IntegrationTest [ ['virtual_machines', nil, 'Host name', 'testvm2.shell'], - ['repositories', 'Add new repository', 'It may take a minute or two before you can clone your new repository.', 'active/foo'], + ['/repositories', 'Add new repository', 'It may take a minute or two before you can clone your new repository.', 'active/foo'], ['/current_token', nil, 'HISTIGNORE=$HISTIGNORE', 'ARVADOS_API_TOKEN=3kg6k6lzmp9kj5'], ['ssh_keys', 'Add new SSH key', 'Click here to learn about SSH keys in Arvados.', 'active'], ].each do |page_name, button_name, look_for, content| test "test user settings menu for page #{page_name}" do - if page_name == '/current_token' + if page_name == '/current_token' || page_name == '/repositories' visit page_with_token('active', page_name) else visit page_with_token('active', "/users/#{api_fixture('users')['active']['uuid']}/#{page_name}") @@ -216,7 +216,7 @@ class UserSettingsMenuTest < ActionDispatch::IntegrationTest [ ['virtual_machines', 'You do not have access to any virtual machines.'], - ['/repositories', api_fixture('repositories')['arvados']['uuid']], + ['/repositories', api_fixture('repositories')['arvados']['name']], ['/current_token', 'HISTIGNORE=$HISTIGNORE'], ['ssh_keys', 'You have not yet set up an SSH public key for use with Arvados.'], ].each do |page_name, look_for| diff --git a/build/check-copyright-notices b/build/check-copyright-notices index cf4e9bf181..5298371bd1 100755 --- a/build/check-copyright-notices +++ b/build/check-copyright-notices @@ -173,7 +173,7 @@ ${cc} ${cc}${cc:+ }SPDX-License-Identifier: CC-BY-SA-3.0${ce}" found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*Arvados' || true) case ${fnm} in - Makefile | build/* | lib/* | tools/* | apps/* | services/*) + Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job) want=${wantGPL} ;; crunch_scripts/* | backports/* | docker/* | sdk/*) diff --git a/by-sa-3.0.txt b/cc-by-sa-3.0.txt similarity index 100% rename from by-sa-3.0.txt rename to cc-by-sa-3.0.txt diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance index bccfdac0b8..b66e9c0526 100755 --- a/sdk/cli/bin/arv-run-pipeline-instance +++ b/sdk/cli/bin/arv-run-pipeline-instance @@ -260,31 +260,46 @@ class JobCache [] end end + + # create() returns [job, exception]. If both job and exception are + # nil, there was a non-retryable error and the call should not be + # attempted again. def self.create(pipeline, component, job, create_params) @cache ||= {} body = {job: no_nil_values(job)}.merge(no_nil_values(create_params)) - result = $client.execute(:api_method => $arvados.jobs.create, - :body_object => body, - :authenticated => false, - :headers => { - authorization: 'OAuth2 '+$arv.config['ARVADOS_API_TOKEN'] - }) - j = JSON.parse result.body, :symbolize_names => true - if j.is_a? Hash and j[:uuid] + result = nil + begin + result = $client.execute( + :api_method => $arvados.jobs.create, + :body_object => body, + :authenticated => false, + :headers => { + authorization: 'OAuth2 '+$arv.config['ARVADOS_API_TOKEN'] + }) + if result.status == 429 || result.status >= 500 + raise Exception.new("HTTP status #{result.status}") + end + rescue Exception => e + return nil, e + end + j = JSON.parse(result.body, :symbolize_names => true) rescue nil + if result.status == 200 && j.is_a?(Hash) && j[:uuid] @cache[j[:uuid]] = j + return j, nil else - debuglog "create job: #{j[:errors] rescue nil} with attributes #{body}", 0 + errors = j[:errors] rescue [] + debuglog "create job: [#{result.status}] #{errors.inspect} with attributes #{body}", 0 msg = "" - j[:errors].each do |err| + errors.each do |err| msg += "Error creating job for component #{component}: #{err}\n" end msg += "Job submission was: #{body.to_json}" pipeline.log_stderr(msg) - nil + return nil, nil end end @@ -396,7 +411,10 @@ class WhRunPipelineInstance end end if !errors.empty? - abort "\n#{Time.now} -- pipeline_template #{@template[:uuid]}\nErrors:\n#{errors.collect { |c,p,e| "#{c}::#{p} - #{e}\n" }.join ""}" + all_errors = errors.collect do |c,p,e| + "#{c}::#{p} - #{e}\n" + end.join("") + abort "\n#{Time.now} -- pipeline_template #{@template[:uuid]}\nErrors:\n#{all_errors}" end debuglog "options=" + @options.pretty_inspect self @@ -463,7 +481,7 @@ class WhRunPipelineInstance # are fully specified (any output_of script_parameters are resolved # to real value) my_submit_id = "instance #{@instance[:uuid]} rand #{rand(2**64).to_s(36)}" - job = JobCache.create(@instance, cname, { + job, err = JobCache.create(@instance, cname, { :script => c[:script], :script_parameters => Hash[c[:script_parameters].map do |key, spec| [key, spec[:value]] @@ -490,9 +508,11 @@ class WhRunPipelineInstance c[:job] = job c[:run_in_process] = (@options[:run_jobs_here] and job[:submit_id] == my_submit_id) - else + elsif err.nil? debuglog "component #{cname} new job failed", 0 job_creation_failed += 1 + else + debuglog "component #{cname} new job failed, err=#{err}", 0 end end @@ -657,7 +677,7 @@ class WhRunPipelineInstance @instance[:state] = 'Complete' else @instance[:state] = 'Paused' - end + end else if ended == @components.length or failed > 0 @instance[:state] = success ? 'Complete' : 'Failed' diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index c0e3075c9b..afca52cb73 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -1,7 +1,7 @@ #!/usr/bin/env perl # Copyright (C) The Arvados Authors. All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: AGPL-3.0 # -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*- diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 4584d955e6..52009dadab 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -424,7 +424,7 @@ class ArvCwlRunner(object): if kwargs.get("submit"): # Submit a runner job to run the workflow for us. if self.work_api == "containers": - if tool.tool["class"] == "CommandLineTool": + if tool.tool["class"] == "CommandLineTool" and kwargs.get("wait"): kwargs["runnerjob"] = tool.tool["id"] upload_dependencies(self, kwargs["name"], diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh index 88860c04eb..236658c1f1 100755 --- a/sdk/cwl/test_with_arvbox.sh +++ b/sdk/cwl/test_with_arvbox.sh @@ -48,6 +48,8 @@ if test -z "$ARVBOX_CONTAINER" ; then fi if test $reset_container = 1 ; then + arvbox stop + docker rm $ARVBOX_CONTAINER arvbox reset -f fi @@ -81,7 +83,7 @@ export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token) if test "$tag" = "latest" ; then arv-keepdocker --pull arvados/jobs $tag else - jobsimg=$(curl http://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])") + jobsimg=\$(curl http://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])") arv-keepdocker --pull arvados/jobs $jobsimg docker tag -f arvados/jobs:$jobsimg arvados/jobs:latest arv-keepdocker arvados/jobs latest diff --git a/sdk/cwl/tests/noreuse.cwl b/sdk/cwl/tests/noreuse.cwl index 46771d1101..4c95eb6817 100644 --- a/sdk/cwl/tests/noreuse.cwl +++ b/sdk/cwl/tests/noreuse.cwl @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + cwlVersion: v1.0 class: Workflow $namespaces: diff --git a/sdk/cwl/tests/stdout.cwl b/sdk/cwl/tests/stdout.cwl index c76a6edec4..2100c37ff1 100644 --- a/sdk/cwl/tests/stdout.cwl +++ b/sdk/cwl/tests/stdout.cwl @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + cwlVersion: v1.0 class: CommandLineTool baseCommand: echo diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index f571bdbfdd..57efb97c48 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -497,14 +497,15 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): 'keep_disk': {'keep_service_uuid': svc['uuid'] } }).execute() - # If keepproxy is running, send SIGHUP to make it discover the new - # keepstore services. - proxypidfile = _pidfile('keepproxy') - if os.path.exists(proxypidfile): - try: - os.kill(int(open(proxypidfile).read()), signal.SIGHUP) - except OSError: - os.remove(proxypidfile) + # If keepproxy and/or keep-web is running, send SIGHUP to make + # them discover the new keepstore services. + for svc in ('keepproxy', 'keep-web'): + pidfile = _pidfile('keepproxy') + if os.path.exists(pidfile): + try: + os.kill(int(open(pidfile).read()), signal.SIGHUP) + except OSError: + os.remove(pidfile) def _stop_keep(n): kill_server_pid(_pidfile('keep{}'.format(n))) diff --git a/services/api/Rakefile b/services/api/Rakefile index 3136cdd9af..fad803cb94 100644 --- a/services/api/Rakefile +++ b/services/api/Rakefile @@ -43,7 +43,7 @@ namespace :db do origfile = File.new origfnm origfile.each_line do |line| if !copyright_done - if !/Copyright .* Arvados/ + if !/Copyright .* Arvados/.match(line) tmpfile.write "-- Copyright (C) The Arvados Authors. All rights reserved.\n--\n-- SPDX-License-Identifier: AGPL-3.0\n\n" end copyright_done = true diff --git a/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb new file mode 100644 index 0000000000..2c90c9a04b --- /dev/null +++ b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb @@ -0,0 +1,21 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +require 'migrate_yaml_to_json' + +class JobsYamlToJson < ActiveRecord::Migration + def up + [ + 'components', + 'script_parameters', + 'runtime_constraints', + 'tasks_summary', + ].each do |column| + MigrateYAMLToJSON.migrate("jobs", column) + end + end + + def down + end +end diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql index b38cba8109..7e6bb1dca4 100644 --- a/services/api/db/structure.sql +++ b/services/api/db/structure.sql @@ -2789,3 +2789,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170419173712'); INSERT INTO schema_migrations (version) VALUES ('20170419175801'); +INSERT INTO schema_migrations (version) VALUES ('20170628185847'); + diff --git a/services/api/lib/migrate_yaml_to_json.rb b/services/api/lib/migrate_yaml_to_json.rb new file mode 100644 index 0000000000..1db7ed0113 --- /dev/null +++ b/services/api/lib/migrate_yaml_to_json.rb @@ -0,0 +1,32 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +module MigrateYAMLToJSON + def self.migrate(table, column) + conn = ActiveRecord::Base.connection + n = conn.update( + "UPDATE #{table} SET #{column}=$1 WHERE #{column}=$2", + "#{table}.#{column} convert YAML to JSON", + [[nil, "{}"], [nil, "--- {}\n"]]) + Rails.logger.info("#{table}.#{column}: #{n} rows updated using empty hash") + finished = false + while !finished + n = 0 + conn.exec_query( + "SELECT id, #{column} FROM #{table} WHERE #{column} LIKE $1 LIMIT 100", + "#{table}.#{column} check for YAML", + [[nil, '---%']], + ).rows.map do |id, yaml| + n += 1 + json = SafeJSON.dump(YAML.load(yaml)) + conn.exec_query( + "UPDATE #{table} SET #{column}=$1 WHERE id=$2 AND #{column}=$3", + "#{table}.#{column} convert YAML to JSON", + [[nil, json], [nil, id], [nil, yaml]]) + end + Rails.logger.info("#{table}.#{column}: #{n} rows updated") + finished = (n == 0) + end + end +end diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 0361ffeab6..30ae6b40e0 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -511,6 +511,8 @@ class CollectionDirectory(CollectionDirectoryBase): self.collection.stop_threads() def clear(self): + if self.collection is not None: + self.collection.stop_threads() super(CollectionDirectory, self).clear() self._manifest_size = 0 @@ -677,6 +679,7 @@ class TagsDirectory(Directory): self.num_retries = num_retries self._poll = True self._poll_time = poll_time + self._extra = set() def want_event_subscribe(self): return True @@ -685,15 +688,41 @@ class TagsDirectory(Directory): def update(self): with llfuse.lock_released: tags = self.api.links().list( - filters=[['link_class', '=', 'tag']], - select=['name'], distinct=True + filters=[['link_class', '=', 'tag'], ["name", "!=", ""]], + select=['name'], distinct=True, limit=1000 ).execute(num_retries=self.num_retries) if "items" in tags: - self.merge(tags['items'], + self.merge(tags['items']+[{"name": n} for n in self._extra], lambda i: i['name'], lambda a, i: a.tag == i['name'], lambda i: TagDirectory(self.inode, self.inodes, self.api, self.num_retries, i['name'], poll=self._poll, poll_time=self._poll_time)) + @use_counter + @check_update + def __getitem__(self, item): + if super(TagsDirectory, self).__contains__(item): + return super(TagsDirectory, self).__getitem__(item) + with llfuse.lock_released: + tags = self.api.links().list( + filters=[['link_class', '=', 'tag'], ['name', '=', item]], limit=1 + ).execute(num_retries=self.num_retries) + if tags["items"]: + self._extra.add(item) + self.update() + return super(TagsDirectory, self).__getitem__(item) + + @use_counter + @check_update + def __contains__(self, k): + if super(TagsDirectory, self).__contains__(k): + return True + try: + self[k] + return True + except KeyError: + pass + return False + class TagDirectory(Directory): """A special directory that contains as subdirectories all collections visible diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index 174e370c13..d72effc075 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -170,13 +170,14 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo expire: exp, pdh: collection.PortableDataHash, }) - c.collections.Add(collection.PortableDataHash, &cachedCollection{ - expire: exp, - collection: collection, - }) - if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { - go c.pruneCollections() - } + // Disabled, see #11945 + // c.collections.Add(collection.PortableDataHash, &cachedCollection{ + // expire: exp, + // collection: collection, + // }) + // if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { + // go c.pruneCollections() + // } return collection, nil } diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go index 77eaf0cd0b..05325270e6 100644 --- a/services/keep-web/cache_test.go +++ b/services/keep-web/cache_test.go @@ -11,6 +11,8 @@ import ( ) func (s *UnitSuite) TestCache(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) @@ -72,6 +74,8 @@ func (s *UnitSuite) TestCache(c *check.C) { } func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) @@ -90,6 +94,8 @@ func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { } func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) diff --git a/services/ws/session_v0_test.go b/services/ws/session_v0_test.go index 0ae8ceb1c4..1213be5d14 100644 --- a/services/ws/session_v0_test.go +++ b/services/ws/session_v0_test.go @@ -83,28 +83,31 @@ func (s *v0Suite) TestLastLogID(c *check.C) { }), check.IsNil) s.expectStatus(c, r, 200) + avoidRace := make(chan struct{}, cap(uuidChan)) go func() { + // When last_log_id is given, although v0session sends + // old events in order, and sends new events in order, + // it doesn't necessarily finish sending all old + // events before sending any new events. To avoid + // hitting this bug in the test, we wait for the old + // events to arrive before emitting any new events. + <-avoidRace s.emitEvents(uuidChan) close(uuidChan) }() - done := make(chan bool) go func() { for uuid := range uuidChan { for _, etype := range []string{"create", "blip", "update"} { lg := s.expectLog(c, r) - c.Check(lg.ObjectUUID, check.Equals, uuid) + for lg.ObjectUUID != uuid { + lg = s.expectLog(c, r) + } c.Check(lg.EventType, check.Equals, etype) } + avoidRace <- struct{}{} } - close(done) }() - - select { - case <-time.After(10 * time.Second): - c.Fatal("timeout") - case <-done: - } } func (s *v0Suite) TestPermission(c *check.C) { @@ -117,16 +120,21 @@ func (s *v0Suite) TestPermission(c *check.C) { }), check.IsNil) s.expectStatus(c, r, 200) - uuidChan := make(chan string, 1) + uuidChan := make(chan string, 2) go func() { s.token = arvadostest.AdminToken - s.emitEvents(nil) + s.emitEvents(uuidChan) s.token = arvadostest.ActiveToken s.emitEvents(uuidChan) }() + wrongUUID := <-uuidChan + rightUUID := <-uuidChan lg := s.expectLog(c, r) - c.Check(lg.ObjectUUID, check.Equals, <-uuidChan) + for lg.ObjectUUID != rightUUID { + c.Check(lg.ObjectUUID, check.Not(check.Equals), wrongUUID) + lg = s.expectLog(c, r) + } } func (s *v0Suite) TestSendBadJSON(c *check.C) { @@ -170,7 +178,9 @@ func (s *v0Suite) TestSubscribe(c *check.C) { for _, etype := range []string{"create", "blip", "update"} { lg := s.expectLog(c, r) - c.Check(lg.ObjectUUID, check.Equals, uuid) + for lg.ObjectUUID != uuid { + lg = s.expectLog(c, r) + } c.Check(lg.EventType, check.Equals, etype) } } @@ -228,8 +238,17 @@ func (s *v0Suite) expectStatus(c *check.C, r *json.Decoder, status int) { func (s *v0Suite) expectLog(c *check.C, r *json.Decoder) *arvados.Log { lg := &arvados.Log{} - c.Check(r.Decode(lg), check.IsNil) - return lg + ok := make(chan struct{}) + go func() { + c.Check(r.Decode(lg), check.IsNil) + close(ok) + }() + select { + case <-time.After(10 * time.Second): + panic("timed out") + case <-ok: + return lg + } } func (s *v0Suite) testClient() (*server, *websocket.Conn, *json.Decoder, *json.Encoder) {