The API server publishes a machine-readable description of its endpoints and some additional site configuration values via a JSON-formatted discovery document. This is available at @/discovery/v1/apis/arvados/v1/rest@, for example @https://{{ site.arvados_api_host }}/discovery/v1/apis/arvados/v1/rest@. Some Arvados SDKs use the discovery document to generate language bindings.
+h2. Exported configuration
+
+The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config@. The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
+
h2. Workbench examples
Many Arvados Workbench pages, under the the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
github.com/Microsoft/go-winio v0.4.5 // indirect
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
- github.com/arvados/cgofuse v1.2.0
+ github.com/arvados/cgofuse v1.2.0-arvados1
github.com/aws/aws-sdk-go v1.25.30
github.com/coreos/go-oidc v2.1.0+incompatible
github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/arvados/cgofuse v1.2.0 h1:sWgVxyvSFjH965Uc7ReScn/cBl9Jemc9SeUNlEmjRH4=
-github.com/arvados/cgofuse v1.2.0/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
+github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQkfoEA2HqE=
+github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=
"Login.ProviderAppSecret": false,
"Login.LoginCluster": true,
"Login.RemoteTokenRefresh": true,
- "Mail": false,
+ "Mail": true,
+ "Mail.MailchimpAPIKey": false,
+ "Mail.MailchimpListID": false,
+ "Mail.SendUserSetupNotificationEmail": false,
+ "Mail.IssueReporterEmailFrom": false,
+ "Mail.IssueReporterEmailTo": false,
+ "Mail.SupportEmailAddress": true,
+ "Mail.EmailFrom": false,
"ManagementToken": false,
"PostgreSQL": false,
"RemoteClusters": true,
// A QueueEnt is an entry in the queue, consisting of a container
// record and the instance type that should be used to run it.
type QueueEnt struct {
- // The container to run. Only the UUID, State, Priority, and
- // RuntimeConstraints fields are populated.
+ // The container to run. Only the UUID, State, Priority,
+ // RuntimeConstraints, Mounts, and ContainerImage fields are
+ // populated.
Container arvados.Container `json:"container"`
InstanceType arvados.InstanceType `json:"instance_type"`
}
*next[upd.UUID] = upd
}
}
- selectParam := []string{"uuid", "state", "priority", "runtime_constraints"}
+ selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts"}
limitParam := 1000
mine, err := cq.fetchAll(arvados.ResourceListParams{
func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
errorTypeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
+ // Make sure the relevant container fields are
+ // actually populated.
+ c.Check(ctr.ContainerImage, check.Equals, "test")
+ c.Check(ctr.RuntimeConstraints.VCPUs, check.Equals, 4)
+ c.Check(ctr.RuntimeConstraints.RAM, check.Equals, int64(12000000000))
+ c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
+ c.Check(ctr.Mounts["/var/spool/cwl"].Capacity, check.Equals, int64(24000000000))
return arvados.InstanceType{}, errors.New("no suitable instance type")
}
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.add_runtime_dependency 'andand', '~> 1.3', '>= 1.3.3'
s.add_runtime_dependency 'oj', '~> 3.0'
s.add_runtime_dependency 'curb', '~> 0.8'
+ s.add_runtime_dependency 'launchy', '< 2.5'
# arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
s.add_dependency('faraday', '< 0.16')
s.homepage =
import re
import pkg_resources # part of setuptools
+### begin monkey patch ###
+# Monkey patch solution for bug #16169
+#
+# There is a bug in upstream cwltool where the version updater needs
+# to replace the document fragments in the loader index with the
+# updated ones, but actually it only does it for the root document.
+# Normally we just fix the bug in upstream but that's challenging
+# because current cwltool dropped support for Python 2.7 and we're
+# still supporting py2 in Arvados 2.0 (although py2 support will most
+# likely be dropped in Arvados 2.1). Making a bugfix fork comes with
+# its own complications (it would need to be added to PyPi) so monkey
+# patching is the least disruptive fix (and is relatively safe because
+# our cwltool dependency is pinned to a specific version). This
+# should be removed as soon as a bugfix goes into upstream cwltool and
+# we upgrade to it.
+#
+import cwltool.load_tool
+from cwltool.utils import visit_class
+from six.moves import urllib
+original_resolve_and_validate_document = cwltool.load_tool.resolve_and_validate_document
+def wrapped_resolve_and_validate_document(
+ loadingContext, # type: LoadingContext
+ workflowobj, # type: Union[CommentedMap, CommentedSeq]
+ uri, # type: Text
+ preprocess_only=False, # type: bool
+ skip_schemas=None, # type: Optional[bool]
+ ):
+ loadingContext, uri = original_resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only, skip_schemas)
+ if loadingContext.do_update in (True, None):
+ fileuri = urllib.parse.urldefrag(uri)[0]
+ def update_index(pr):
+ loadingContext.loader.idx[pr["id"]] = pr
+ visit_class(loadingContext.loader.idx[fileuri], ("CommandLineTool", "Workflow", "ExpressionTool"), update_index)
+ return loadingContext, uri
+cwltool.load_tool.resolve_and_validate_document = wrapped_resolve_and_validate_document
+### end monkey patch ###
+
from schema_salad.sourceline import SourceLine
import schema_salad.validate as validate
import cwltool.main
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+requirements:
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+inputs:
+ d: Directory
+steps:
+ step1:
+ in:
+ d: d
+ out: [out]
+ run: wf/16169-step.cwl
+outputs:
+ out:
+ type: File
+ outputSource: step1/out
should_fail: true
tool: 15295-bad-keep-ref.cwl
doc: Test checking for invalid keepref
+
+- job: listing-job.yml
+ output: {
+ "out": {
+ "class": "File",
+ "location": "output.txt",
+ "size": 5,
+ "checksum": "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1"
+ }
+ }
+ tool: 16169-no-listing-hint.cwl
+ doc: "Test cwltool:LoadListingRequirement propagation"
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ InlineJavascriptRequirement: {}
+ DockerRequirement:
+ dockerPull: debian:stretch-slim
+inputs:
+ d: Directory
+outputs:
+ out: stdout
+stdout: output.txt
+arguments:
+ [echo, "${if(inputs.d.listing === undefined) {return 'true';} else {return 'false';}}"]
type ContainerRequestState string
const (
- ContainerRequestStateUncomitted = ContainerState("Uncommitted")
- ContainerRequestStateCommitted = ContainerState("Committed")
- ContainerRequestStateFinal = ContainerState("Final")
+ ContainerRequestStateUncomitted = ContainerRequestState("Uncommitted")
+ ContainerRequestStateCommitted = ContainerRequestState("Committed")
+ ContainerRequestStateFinal = ContainerRequestState("Final")
)
try:
image_hash = find_one_image_hash(args.image, args.tag)
except DockerError as error:
- logger.error(error.message)
+ logger.error(str(error))
sys.exit(1)
if not docker_image_compatible(api, image_hash):
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
begin
user = User.register(authinfo)
rescue => e
- Rails.logger.warn e
+ Rails.logger.warn "User.register error #{e}"
+ Rails.logger.warn "authinfo was #{authinfo.inspect}"
return redirect_to login_failure_url
end
user = self
redirects = 0
while (uuid = user.redirect_to_user_uuid)
- user = User.unscoped.find_by_uuid(uuid)
- if !user
- raise Exception.new("user uuid #{user.uuid} redirects to nonexistent uuid #{uuid}")
+ break if uuid.empty?
+ nextuser = User.unscoped.find_by_uuid(uuid)
+ if !nextuser
+ raise Exception.new("user uuid #{user.uuid} redirects to nonexistent uuid '#{uuid}'")
end
+ user = nextuser
redirects += 1
if redirects > 15
raise "Starting from #{self.uuid} redirect_to_user_uuid exceeded maximum number of redirects"
*.gem
+Gemfile.lock
\ No newline at end of file
+++ /dev/null
-PATH
- remote: .
- specs:
- arvados-login-sync (1.5.0.dev20200118023341)
- arvados (~> 1.3.0, >= 1.3.0)
- faraday (< 0.16)
- signet (< 0.12)
-
-GEM
- remote: https://rubygems.org/
- specs:
- activesupport (5.0.7.2)
- concurrent-ruby (~> 1.0, >= 1.0.2)
- i18n (>= 0.7, < 2)
- minitest (~> 5.1)
- tzinfo (~> 1.1)
- addressable (2.7.0)
- public_suffix (>= 2.0.2, < 5.0)
- andand (1.3.3)
- arvados (1.3.3.20190320201707)
- activesupport (>= 3)
- andand (~> 1.3, >= 1.3.3)
- arvados-google-api-client (>= 0.7, < 0.8.9)
- i18n (~> 0)
- json (>= 1.7.7, < 3)
- jwt (>= 0.1.5, < 2)
- arvados-google-api-client (0.8.7.3)
- activesupport (>= 3.2, < 5.1)
- addressable (~> 2.3)
- autoparse (~> 0.3)
- extlib (~> 0.9)
- faraday (~> 0.9)
- googleauth (~> 0.3)
- launchy (~> 2.4)
- multi_json (~> 1.10)
- retriable (~> 1.4)
- signet (~> 0.6)
- autoparse (0.3.3)
- addressable (>= 2.3.1)
- extlib (>= 0.9.15)
- multi_json (>= 1.0.0)
- concurrent-ruby (1.1.5)
- extlib (0.9.16)
- faraday (0.15.4)
- multipart-post (>= 1.2, < 3)
- googleauth (0.9.0)
- faraday (~> 0.12)
- jwt (>= 1.4, < 3.0)
- memoist (~> 0.16)
- multi_json (~> 1.11)
- os (>= 0.9, < 2.0)
- signet (~> 0.7)
- i18n (0.9.5)
- concurrent-ruby (~> 1.0)
- json (2.3.0)
- jwt (1.5.6)
- launchy (2.4.3)
- addressable (~> 2.3)
- memoist (0.16.2)
- metaclass (0.0.4)
- minitest (5.11.3)
- mocha (1.8.0)
- metaclass (~> 0.0.1)
- multi_json (1.14.1)
- multipart-post (2.1.1)
- os (1.0.1)
- public_suffix (4.0.3)
- rake (12.3.2)
- retriable (1.4.1)
- signet (0.11.0)
- addressable (~> 2.3)
- faraday (~> 0.9)
- jwt (>= 1.5, < 3.0)
- multi_json (~> 1.10)
- thread_safe (0.3.6)
- tzinfo (1.2.6)
- thread_safe (~> 0.1)
-
-PLATFORMS
- ruby
-
-DEPENDENCIES
- arvados-login-sync!
- minitest (>= 5.0.0)
- mocha (>= 1.5.0)
- rake
-
-BUNDLED WITH
- 1.17.3
ENV["GIT_DIR"] = File.expand_path "#{__dir__}/../../.git"
ENV["GIT_WORK_TREE"] = File.expand_path "#{__dir__}/../.."
git_timestamp, git_hash = `git log -n1 --first-parent --format=%ct:%H #{__dir__}`.chomp.split(":")
- version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ if ENV["ARVADOS_BUILDING_VERSION"]
+ version = ENV["ARVADOS_BUILDING_VERSION"]
+ else
+ version = `#{__dir__}/../../build/version-at-commit.sh #{git_hash}`.encode('utf-8').strip
+ end
git_timestamp = Time.at(git_timestamp.to_i).utc
ensure
ENV["GIT_DIR"] = git_dir
s.executables << "arvados-login-sync"
s.required_ruby_version = '>= 2.1.0'
s.add_runtime_dependency 'arvados', '~> 1.3.0', '>= 1.3.0'
+ s.add_runtime_dependency 'launchy', '< 2.5'
# arvados-google-api-client 0.8.7.2 is incompatible with faraday 0.16.2
s.add_dependency('faraday', '< 0.16')
# arvados-google-api-client (and thus arvados) gems
projects / arvados.git / commitdiff