From: Lucas Di Pentima Date: Mon, 10 Jul 2017 13:22:44 +0000 (-0300) Subject: 7475: Merge branch 'master' into 7475-nodemgr-unsatisfiable-job-comms X-Git-Tag: 1.1.0~104^2~3 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/17cd77ac947e2c8f4ca51aa930ffc235051d7f72?hp=f507162f3974797b741a0f740b407daefceab0b6 7475: Merge branch 'master' into 7475-nodemgr-unsatisfiable-job-comms Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- diff --git a/.licenseignore b/.licenseignore index 1d0356b51e..51980b16c2 100644 --- a/.licenseignore +++ b/.licenseignore @@ -1,7 +1,9 @@ *agpl-3.0.html *agpl-3.0.txt +apache-2.0.txt apps/workbench/app/assets/javascripts/list.js apps/workbench/public/webshell/* +AUTHORS */bootstrap.css */bootstrap.js *bootstrap-theme.css diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000000..ad72aea0aa --- /dev/null +++ b/AUTHORS @@ -0,0 +1,18 @@ +# Names should be added to this file with this pattern: +# +# For individuals: +# Name +# +# For organizations: +# Organization +# +# See python fnmatch module documentation for more information. + +Curoverse, Inc. <*@curoverse.com> +Adam Savitzky +Colin Nolan +David +Guillermo Carrasco +Joshua Randall +President and Fellows of Harvard College <*@harvard.edu> +Thomas Mooney diff --git a/COPYING b/COPYING index 2cba2ad20a..61c31397a0 100644 --- a/COPYING +++ b/COPYING @@ -1,15 +1,19 @@ -Server-side components of Arvados contained in the apps/ and services/ -directories, including the API Server, Workbench, and Crunch, are licensed -under the GNU Affero General Public License version 3 (see agpl-3.0.txt). +Unless indicated otherwise in the header of the file, the files in this +repository are distributed under one of three different licenses: AGPL-3.0, +Apache-2.0 or CC-BY-SA-3.0. -The files and directories under the build/, lib/ and tools/ directories are -licensed under the GNU Affero General Public License version 3 (see -agpl-3.0.txt). +Individual files contain an SPDX tag that indicates the license for the file. +These are the three tags in use: -The Arvados client Software Development Kits contained in the sdk/ directory, -example scripts in the crunch_scripts/ directory, the files and directories -under backports/ and docker/, and code samples in the Aravados documentation -are licensed under the Apache License, Version 2.0 (see LICENSE-2.0.txt). + SPDX-License-Identifier: AGPL-3.0 + SPDX-License-Identifier: Apache-2.0 + SPDX-License-Identifier: CC-BY-SA-3.0 -The Arvados Documentation located in the doc/ directory is licensed under the -Creative Commons Attribution-Share Alike 3.0 United States (see by-sa-3.0.txt). +This enables machine processing of license information based on the SPDX +License Identifiers that are available here: http://spdx.org/licenses/ + +The full license text for each license is available in this directory: + + AGPL-3.0: agpl-3.0.txt + Apache-2.0: apache-2.0.txt + CC-BY-SA-3.0: cc-by-sa-3.0.txt diff --git a/LICENSE-2.0.txt b/apache-2.0.txt similarity index 100% rename from LICENSE-2.0.txt rename to apache-2.0.txt diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb index 99065947b8..f8fcf5108f 100644 --- a/apps/workbench/app/controllers/collections_controller.rb +++ b/apps/workbench/app/controllers/collections_controller.rb @@ -115,6 +115,10 @@ class CollectionsController < ApplicationController end def show_file_links + if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url + # show_file will redirect to keep-web's directory listing + return show_file + end Thread.current[:reader_tokens] = [params[:reader_token]] return if false.equal?(find_object_by_uuid) render layout: false @@ -283,7 +287,12 @@ class CollectionsController < ApplicationController helper_method :download_link def download_link - collections_url + "/download/#{@object.uuid}/#{@search_sharing.first.api_token}/" + token = @search_sharing.first.api_token + if Rails.configuration.keep_web_url || Rails.configuration.keep_web_download_url + keep_web_url(@object.uuid, nil, {path_token: token}) + else + collections_url + "/download/#{@object.uuid}/#{token}/" + end end def share @@ -444,7 +453,7 @@ class CollectionsController < ApplicationController uri.path += 't=' + opts[:path_token] + '/' end uri.path += '_/' - uri.path += URI.escape(file) + uri.path += URI.escape(file) if file query = Hash[URI.decode_www_form(uri.query || '')] { query_token: 'api_token', diff --git a/apps/workbench/app/views/collections/_show_tags.html.erb b/apps/workbench/app/views/collections/_show_tags.html.erb index fd734c82dc..afab5266e9 100644 --- a/apps/workbench/app/views/collections/_show_tags.html.erb +++ b/apps/workbench/app/views/collections/_show_tags.html.erb @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0 %>
<% if object.editable? %>

- Edit + Edit

<% end %> diff --git a/apps/workbench/test/integration/collection_upload_test.rb b/apps/workbench/test/integration/collection_upload_test.rb index e54a5c2185..608cd521de 100644 --- a/apps/workbench/test/integration/collection_upload_test.rb +++ b/apps/workbench/test/integration/collection_upload_test.rb @@ -96,11 +96,11 @@ class CollectionUploadTest < ActionDispatch::IntegrationTest test "Report network error" do need_selenium "to make file uploads work" use_token :admin do - # Even if you somehow do port>2^16, surely nx.example.net won't + # Even if port 0 is a thing, surely nx.example.net won't # respond KeepService.where(service_type: 'proxy').first. update_attributes(service_host: 'nx.example.net', - service_port: 99999) + service_port: 0) end visit page_with_token 'active', sandbox_path diff --git a/apps/workbench/test/integration/collections_test.rb b/apps/workbench/test/integration/collections_test.rb index af8e1313f8..8619858dfe 100644 --- a/apps/workbench/test/integration/collections_test.rb +++ b/apps/workbench/test/integration/collections_test.rb @@ -426,8 +426,6 @@ class CollectionsTest < ActionDispatch::IntegrationTest end test "collection tags tab" do - need_selenium - visit page_with_token('active', '/collections/zzzzz-4zz18-bv31uwvy3neko21') click_link 'Tags' @@ -447,8 +445,6 @@ class CollectionsTest < ActionDispatch::IntegrationTest assert_selector 'a', text: 'Cancel' # add two tags - first('.edit-collection-tags').click - first('.glyphicon-plus').click first('.collection-tag-field-key').click first('.collection-tag-field-key').set('key 1') diff --git a/apps/workbench/test/integration/download_test.rb b/apps/workbench/test/integration/download_test.rb index 37faef9c3d..407458b62b 100644 --- a/apps/workbench/test/integration/download_test.rb +++ b/apps/workbench/test/integration/download_test.rb @@ -8,6 +8,8 @@ require 'helpers/download_helper' class DownloadTest < ActionDispatch::IntegrationTest include KeepWebConfig + @@wrote_test_data = false + setup do use_keep_web_config @@ -17,10 +19,13 @@ class DownloadTest < ActionDispatch::IntegrationTest # Keep data isn't populated by fixtures, so we have to write any # data we expect to read. - ['foo', 'w a z', "Hello world\n"].each do |data| - md5 = `echo -n #{data.shellescape} | arv-put --no-progress --raw -` - assert_match /^#{Digest::MD5.hexdigest(data)}/, md5 - assert $?.success?, $? + if !@@wrote_test_data + ['foo', 'w a z', "Hello world\n"].each do |data| + md5 = `echo -n #{data.shellescape} | arv-put --no-progress --raw -` + assert_match /^#{Digest::MD5.hexdigest(data)}/, md5 + assert $?.success?, $? + end + @@wrote_test_data = true end end @@ -29,7 +34,7 @@ class DownloadTest < ActionDispatch::IntegrationTest uuid_or_pdh = api_fixture('collections')['foo_file'][id_type] token = api_fixture('api_client_authorizations')['active_all_collections']['api_token'] visit "/collections/download/#{uuid_or_pdh}/#{token}/" - within "#collection_files" do + within 'ul' do click_link 'foo' end assert_no_selector 'a' diff --git a/build/check-copyright-notices b/build/check-copyright-notices index cf4e9bf181..5298371bd1 100755 --- a/build/check-copyright-notices +++ b/build/check-copyright-notices @@ -173,7 +173,7 @@ ${cc} ${cc}${cc:+ }SPDX-License-Identifier: CC-BY-SA-3.0${ce}" found=$(head -n20 "$fnm" | egrep -A${grepAfter} -B${grepBefore} 'Copyright.*Arvados' || true) case ${fnm} in - Makefile | build/* | lib/* | tools/* | apps/* | services/*) + Makefile | build/* | lib/* | tools/* | apps/* | services/* | sdk/cli/bin/crunch-job) want=${wantGPL} ;; crunch_scripts/* | backports/* | docker/* | sdk/*) diff --git a/by-sa-3.0.txt b/cc-by-sa-3.0.txt similarity index 100% rename from by-sa-3.0.txt rename to cc-by-sa-3.0.txt diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance index bccfdac0b8..b66e9c0526 100755 --- a/sdk/cli/bin/arv-run-pipeline-instance +++ b/sdk/cli/bin/arv-run-pipeline-instance @@ -260,31 +260,46 @@ class JobCache [] end end + + # create() returns [job, exception]. If both job and exception are + # nil, there was a non-retryable error and the call should not be + # attempted again. def self.create(pipeline, component, job, create_params) @cache ||= {} body = {job: no_nil_values(job)}.merge(no_nil_values(create_params)) - result = $client.execute(:api_method => $arvados.jobs.create, - :body_object => body, - :authenticated => false, - :headers => { - authorization: 'OAuth2 '+$arv.config['ARVADOS_API_TOKEN'] - }) - j = JSON.parse result.body, :symbolize_names => true - if j.is_a? Hash and j[:uuid] + result = nil + begin + result = $client.execute( + :api_method => $arvados.jobs.create, + :body_object => body, + :authenticated => false, + :headers => { + authorization: 'OAuth2 '+$arv.config['ARVADOS_API_TOKEN'] + }) + if result.status == 429 || result.status >= 500 + raise Exception.new("HTTP status #{result.status}") + end + rescue Exception => e + return nil, e + end + j = JSON.parse(result.body, :symbolize_names => true) rescue nil + if result.status == 200 && j.is_a?(Hash) && j[:uuid] @cache[j[:uuid]] = j + return j, nil else - debuglog "create job: #{j[:errors] rescue nil} with attributes #{body}", 0 + errors = j[:errors] rescue [] + debuglog "create job: [#{result.status}] #{errors.inspect} with attributes #{body}", 0 msg = "" - j[:errors].each do |err| + errors.each do |err| msg += "Error creating job for component #{component}: #{err}\n" end msg += "Job submission was: #{body.to_json}" pipeline.log_stderr(msg) - nil + return nil, nil end end @@ -396,7 +411,10 @@ class WhRunPipelineInstance end end if !errors.empty? - abort "\n#{Time.now} -- pipeline_template #{@template[:uuid]}\nErrors:\n#{errors.collect { |c,p,e| "#{c}::#{p} - #{e}\n" }.join ""}" + all_errors = errors.collect do |c,p,e| + "#{c}::#{p} - #{e}\n" + end.join("") + abort "\n#{Time.now} -- pipeline_template #{@template[:uuid]}\nErrors:\n#{all_errors}" end debuglog "options=" + @options.pretty_inspect self @@ -463,7 +481,7 @@ class WhRunPipelineInstance # are fully specified (any output_of script_parameters are resolved # to real value) my_submit_id = "instance #{@instance[:uuid]} rand #{rand(2**64).to_s(36)}" - job = JobCache.create(@instance, cname, { + job, err = JobCache.create(@instance, cname, { :script => c[:script], :script_parameters => Hash[c[:script_parameters].map do |key, spec| [key, spec[:value]] @@ -490,9 +508,11 @@ class WhRunPipelineInstance c[:job] = job c[:run_in_process] = (@options[:run_jobs_here] and job[:submit_id] == my_submit_id) - else + elsif err.nil? debuglog "component #{cname} new job failed", 0 job_creation_failed += 1 + else + debuglog "component #{cname} new job failed, err=#{err}", 0 end end @@ -657,7 +677,7 @@ class WhRunPipelineInstance @instance[:state] = 'Complete' else @instance[:state] = 'Paused' - end + end else if ended == @components.length or failed > 0 @instance[:state] = success ? 'Complete' : 'Failed' diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job index c0e3075c9b..afca52cb73 100755 --- a/sdk/cli/bin/crunch-job +++ b/sdk/cli/bin/crunch-job @@ -1,7 +1,7 @@ #!/usr/bin/env perl # Copyright (C) The Arvados Authors. All rights reserved. # -# SPDX-License-Identifier: Apache-2.0 +# SPDX-License-Identifier: AGPL-3.0 # -*- mode: perl; perl-indent-level: 2; indent-tabs-mode: nil; -*- diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py index 4584d955e6..52009dadab 100644 --- a/sdk/cwl/arvados_cwl/__init__.py +++ b/sdk/cwl/arvados_cwl/__init__.py @@ -424,7 +424,7 @@ class ArvCwlRunner(object): if kwargs.get("submit"): # Submit a runner job to run the workflow for us. if self.work_api == "containers": - if tool.tool["class"] == "CommandLineTool": + if tool.tool["class"] == "CommandLineTool" and kwargs.get("wait"): kwargs["runnerjob"] = tool.tool["id"] upload_dependencies(self, kwargs["name"], diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh index 88860c04eb..236658c1f1 100755 --- a/sdk/cwl/test_with_arvbox.sh +++ b/sdk/cwl/test_with_arvbox.sh @@ -48,6 +48,8 @@ if test -z "$ARVBOX_CONTAINER" ; then fi if test $reset_container = 1 ; then + arvbox stop + docker rm $ARVBOX_CONTAINER arvbox reset -f fi @@ -81,7 +83,7 @@ export ARVADOS_API_TOKEN=\$(cat /var/lib/arvados/superuser_token) if test "$tag" = "latest" ; then arv-keepdocker --pull arvados/jobs $tag else - jobsimg=$(curl http://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])") + jobsimg=\$(curl http://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])") arv-keepdocker --pull arvados/jobs $jobsimg docker tag -f arvados/jobs:$jobsimg arvados/jobs:latest arv-keepdocker arvados/jobs latest diff --git a/sdk/cwl/tests/noreuse.cwl b/sdk/cwl/tests/noreuse.cwl index 46771d1101..4c95eb6817 100644 --- a/sdk/cwl/tests/noreuse.cwl +++ b/sdk/cwl/tests/noreuse.cwl @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + cwlVersion: v1.0 class: Workflow $namespaces: diff --git a/sdk/cwl/tests/stdout.cwl b/sdk/cwl/tests/stdout.cwl index c76a6edec4..2100c37ff1 100644 --- a/sdk/cwl/tests/stdout.cwl +++ b/sdk/cwl/tests/stdout.cwl @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + cwlVersion: v1.0 class: CommandLineTool baseCommand: echo diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py index f571bdbfdd..57efb97c48 100644 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@ -497,14 +497,15 @@ def run_keep(blob_signing_key=None, enforce_permissions=False, num_servers=2): 'keep_disk': {'keep_service_uuid': svc['uuid'] } }).execute() - # If keepproxy is running, send SIGHUP to make it discover the new - # keepstore services. - proxypidfile = _pidfile('keepproxy') - if os.path.exists(proxypidfile): - try: - os.kill(int(open(proxypidfile).read()), signal.SIGHUP) - except OSError: - os.remove(proxypidfile) + # If keepproxy and/or keep-web is running, send SIGHUP to make + # them discover the new keepstore services. + for svc in ('keepproxy', 'keep-web'): + pidfile = _pidfile('keepproxy') + if os.path.exists(pidfile): + try: + os.kill(int(open(pidfile).read()), signal.SIGHUP) + except OSError: + os.remove(pidfile) def _stop_keep(n): kill_server_pid(_pidfile('keep{}'.format(n))) diff --git a/services/api/Rakefile b/services/api/Rakefile index 3136cdd9af..fad803cb94 100644 --- a/services/api/Rakefile +++ b/services/api/Rakefile @@ -43,7 +43,7 @@ namespace :db do origfile = File.new origfnm origfile.each_line do |line| if !copyright_done - if !/Copyright .* Arvados/ + if !/Copyright .* Arvados/.match(line) tmpfile.write "-- Copyright (C) The Arvados Authors. All rights reserved.\n--\n-- SPDX-License-Identifier: AGPL-3.0\n\n" end copyright_done = true diff --git a/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb index 705ec55ab1..2c90c9a04b 100644 --- a/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb +++ b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + require 'migrate_yaml_to_json' class JobsYamlToJson < ActiveRecord::Migration diff --git a/services/api/lib/migrate_yaml_to_json.rb b/services/api/lib/migrate_yaml_to_json.rb index fbfc60362c..1db7ed0113 100644 --- a/services/api/lib/migrate_yaml_to_json.rb +++ b/services/api/lib/migrate_yaml_to_json.rb @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + module MigrateYAMLToJSON def self.migrate(table, column) conn = ActiveRecord::Base.connection diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py index 0361ffeab6..30ae6b40e0 100644 --- a/services/fuse/arvados_fuse/fusedir.py +++ b/services/fuse/arvados_fuse/fusedir.py @@ -511,6 +511,8 @@ class CollectionDirectory(CollectionDirectoryBase): self.collection.stop_threads() def clear(self): + if self.collection is not None: + self.collection.stop_threads() super(CollectionDirectory, self).clear() self._manifest_size = 0 @@ -677,6 +679,7 @@ class TagsDirectory(Directory): self.num_retries = num_retries self._poll = True self._poll_time = poll_time + self._extra = set() def want_event_subscribe(self): return True @@ -685,15 +688,41 @@ class TagsDirectory(Directory): def update(self): with llfuse.lock_released: tags = self.api.links().list( - filters=[['link_class', '=', 'tag']], - select=['name'], distinct=True + filters=[['link_class', '=', 'tag'], ["name", "!=", ""]], + select=['name'], distinct=True, limit=1000 ).execute(num_retries=self.num_retries) if "items" in tags: - self.merge(tags['items'], + self.merge(tags['items']+[{"name": n} for n in self._extra], lambda i: i['name'], lambda a, i: a.tag == i['name'], lambda i: TagDirectory(self.inode, self.inodes, self.api, self.num_retries, i['name'], poll=self._poll, poll_time=self._poll_time)) + @use_counter + @check_update + def __getitem__(self, item): + if super(TagsDirectory, self).__contains__(item): + return super(TagsDirectory, self).__getitem__(item) + with llfuse.lock_released: + tags = self.api.links().list( + filters=[['link_class', '=', 'tag'], ['name', '=', item]], limit=1 + ).execute(num_retries=self.num_retries) + if tags["items"]: + self._extra.add(item) + self.update() + return super(TagsDirectory, self).__getitem__(item) + + @use_counter + @check_update + def __contains__(self, k): + if super(TagsDirectory, self).__contains__(k): + return True + try: + self[k] + return True + except KeyError: + pass + return False + class TagDirectory(Directory): """A special directory that contains as subdirectories all collections visible diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go index 174e370c13..d72effc075 100644 --- a/services/keep-web/cache.go +++ b/services/keep-web/cache.go @@ -170,13 +170,14 @@ func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceRelo expire: exp, pdh: collection.PortableDataHash, }) - c.collections.Add(collection.PortableDataHash, &cachedCollection{ - expire: exp, - collection: collection, - }) - if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { - go c.pruneCollections() - } + // Disabled, see #11945 + // c.collections.Add(collection.PortableDataHash, &cachedCollection{ + // expire: exp, + // collection: collection, + // }) + // if int64(len(collection.ManifestText)) > c.MaxCollectionBytes/int64(c.MaxCollectionEntries) { + // go c.pruneCollections() + // } return collection, nil } diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go index 77eaf0cd0b..05325270e6 100644 --- a/services/keep-web/cache_test.go +++ b/services/keep-web/cache_test.go @@ -11,6 +11,8 @@ import ( ) func (s *UnitSuite) TestCache(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) @@ -72,6 +74,8 @@ func (s *UnitSuite) TestCache(c *check.C) { } func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil) @@ -90,6 +94,8 @@ func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { } func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) { + c.Skip("see #11945") + arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, check.Equals, nil)