if hint[:keep_cache]
keep_cache = hint[:keep_cache]
end
+ if hint[:acrContainerImage]
+ attrs['container_image'] = hint[:acrContainerImage]
+ end
end
end
end
assert_text 'script version'
assert_no_selector 'a', text: 'Run this pipeline'
else
- within first('tr[data-kind="arvados#workflow"]') do
+ within 'tr[data-kind="arvados#workflow"]', text: "Workflow with default input specifications" do
click_link 'Show'
end
assert_text "This container request was created from the workflow"
assert_match /Provide a value for .* then click the \"Run\" button to start the workflow/, page.text
end
+
+ test "create workflow with WorkflowRunnerResources" do
+ visit page_with_token('active', '/workflows/zzzzz-7fd4e-validwithinput3')
+
+ find('a,button', text: 'Run this workflow').click
+
+ # Choose project for the container_request being created
+ within('.modal-dialog') do
+ find('.selectable', text: 'A Project').click
+ find('button', text: 'Choose').click
+ end
+ click_link 'Advanced'
+ click_link("API response")
+ assert_text('"container_image": "arvados/jobs:2.0.4"')
+ assert_text('"vcpus": 2')
+ assert_text('"ram": 1293942784')
+ assert_text('"--collection-cache-size=678"')
+
+ end
end
if err != nil {
return err
}
+ err = super.RunProgram(ctx, "services/api", nil, railsEnv, "bundle", "exec", "./script/get_anonymous_user_token.rb")
+ if err != nil {
+ return err
+ }
return nil
}
if cluster.Collections.BlobSigningKey == "" {
cluster.Collections.BlobSigningKey = randomHexString(64)
}
+ if cluster.Users.AnonymousUserToken == "" {
+ cluster.Users.AnonymousUserToken = randomHexString(64)
+ }
+
if super.ClusterType != "production" && cluster.Containers.DispatchPrivateKey == "" {
buf, err := ioutil.ReadFile(filepath.Join(super.SourcePath, "lib", "dispatchcloud", "test", "sshkey_dispatch"))
if err != nil {
}
}
+// Get rpc connection struct initialized to communicate with the
+// specified cluster.
func (s *IntegrationSuite) conn(clusterID string) *rpc.Conn {
return rpc.NewConn(clusterID, s.testClusters[clusterID].controllerURL, true, rpc.PassthroughTokenProvider)
}
+// Return Context, Arvados.Client and keepclient structs initialized
+// to connect to the specified cluster (by clusterID) using with the supplied
+// Arvados token.
func (s *IntegrationSuite) clientsWithToken(clusterID string, token string) (context.Context, *arvados.Client, *keepclient.KeepClient) {
cl := s.testClusters[clusterID].config.Clusters[clusterID]
ctx := auth.NewContext(context.Background(), auth.NewCredentials(token))
return ctx, ac, kc
}
+// Log in as a user called "example", get the user's API token,
+// initialize clients with the API token, set up the user and
+// optionally activate the user. Return client structs for
+// communicating with the cluster on behalf of the 'example' user.
func (s *IntegrationSuite) userClients(rootctx context.Context, c *check.C, conn *rpc.Conn, clusterID string, activate bool) (context.Context, *arvados.Client, *keepclient.KeepClient) {
login, err := conn.UserSessionCreate(rootctx, rpc.UserSessionCreateOptions{
ReturnTo: ",https://example.com",
return ctx, ac, kc
}
+// Return Context, arvados.Client and keepclient structs initialized
+// to communicate with the cluster as the system root user.
func (s *IntegrationSuite) rootClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) {
return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].SystemRootToken)
}
+// Return Context, arvados.Client and keepclient structs initialized
+// to communicate with the cluster as the anonymous user.
+func (s *IntegrationSuite) anonymousClients(clusterID string) (context.Context, *arvados.Client, *keepclient.KeepClient) {
+ return s.clientsWithToken(clusterID, s.testClusters[clusterID].config.Clusters[clusterID].Users.AnonymousUserToken)
+}
+
func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
conn1 := s.conn("z1111")
rootctx1, _, _ := s.rootClients("z1111")
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+func (s *IntegrationSuite) TestGetCollectionAsAnonymous(c *check.C) {
+ conn1 := s.conn("z1111")
+ conn3 := s.conn("z3333")
+ rootctx1, ac1, kc1 := s.rootClients("z1111")
+ userctx3, ac3, _ := s.anonymousClients("z3333")
+
+ // Make sure anonymous token was set
+ c.Assert(ac3.AuthToken, check.Not(check.Equals), "")
+
+ // Create the collection to find its PDH (but don't save it
+ // anywhere yet)
+ var coll1 arvados.Collection
+ fs1, err := coll1.FileSystem(ac1, kc1)
+ c.Assert(err, check.IsNil)
+ f, err := fs1.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
+ c.Assert(err, check.IsNil)
+ _, err = io.WriteString(f, "IntegrationSuite.TestGetCollectionByPDH")
+ c.Assert(err, check.IsNil)
+ err = f.Close()
+ c.Assert(err, check.IsNil)
+ mtxt, err := fs1.MarshalManifest(".")
+ c.Assert(err, check.IsNil)
+ pdh := arvados.PortableDataHash(mtxt)
+
+ // Save the collection on cluster z1111.
+ coll1, err = conn1.CollectionCreate(rootctx1, arvados.CreateOptions{Attrs: map[string]interface{}{
+ "manifest_text": mtxt,
+ }})
+ c.Assert(err, check.IsNil)
+
+ // Share it with the anonymous users group.
+ var outLink arvados.Link
+ err = ac1.RequestAndDecode(&outLink, "POST", "/arvados/v1/links", nil, &arvados.Link{
+ LinkClass: "permission",
+ Name: "can_read",
+ HeadUUID: coll1.UUID,
+ TailUUID: "z1111-j7d0g-anonymouspublic",
+ })
+ c.Check(err, check.IsNil)
+
+ outUser, err := ac3.CurrentUser()
+ c.Check(err, check.IsNil)
+ c.Check(outUser.UUID, check.Equals, "z3333-tpzed-anonymouspublic")
+
+ // Retrieve the collection as anonymous from cluster z3333.
+ coll, err := conn3.CollectionGet(userctx3, arvados.GetOptions{UUID: pdh})
+ c.Check(err, check.IsNil)
+ c.Check(coll.PortableDataHash, check.Equals, pdh)
+}
+
// Get a token from the login cluster (z1111), use it to submit a
// container request on z2222.
func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
type TokenProvider func(context.Context) ([]string, error)
func PassthroughTokenProvider(ctx context.Context) ([]string, error) {
- if incoming, ok := auth.FromContext(ctx); !ok {
+ incoming, ok := auth.FromContext(ctx)
+ if !ok {
return nil, errors.New("no token provided")
- } else {
- return incoming.Tokens, nil
}
+ return incoming.Tokens, nil
}
type Conn struct {
u.User = nil
u.Host = ""
return u.String()
- } else {
- return location
}
+ return location
}
func (conn *Conn) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) {
return err
}
return w.Close()
- } else {
- // Dispatched via crunch-dispatch-slurm. Look up
- // apiserver's node record corresponding to
- // $SLURMD_NODENAME.
- hostname := os.Getenv("SLURMD_NODENAME")
- if hostname == "" {
- hostname, _ = os.Hostname()
- }
- _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) {
- // The "info" field has admin-only info when
- // obtained with a privileged token, and
- // should not be logged.
- node, ok := resp.(map[string]interface{})
- if ok {
- delete(node, "info")
- }
- })
- return err
}
+ // Dispatched via crunch-dispatch-slurm. Look up
+ // apiserver's node record corresponding to
+ // $SLURMD_NODENAME.
+ hostname := os.Getenv("SLURMD_NODENAME")
+ if hostname == "" {
+ hostname, _ = os.Hostname()
+ }
+ _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) {
+ // The "info" field has admin-only info when
+ // obtained with a privileged token, and
+ // should not be logged.
+ node, ok := resp.(map[string]interface{})
+ if ok {
+ delete(node, "info")
+ }
+ })
+ return err
}
func (runner *ContainerRunner) logAPIResponse(label, path string, params map[string]interface{}, munge func(interface{})) (logged bool, err error) {
return runner.token, nil
}
-// UpdateContainerComplete updates the container record state on API
+// UpdateContainerFinal updates the container record state on API
// server to "Complete" or "Cancelled"
func (runner *ContainerRunner) UpdateContainerFinal() error {
update := arvadosclient.Dict{}
"golang.org/x/crypto/ssh"
)
-// Map of available cloud drivers.
+// Drivers is a map of available cloud drivers.
// Clusters.*.Containers.CloudVMs.Driver configuration values
// correspond to keys in this map.
var Drivers = map[string]cloud.Driver{
func boolLabelValue(v bool) string {
if v {
return "1"
- } else {
- return "0"
}
+ return "0"
}
overquota = sorted[i:]
break tryrun
}
+ if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ logger.Info("not locking: crunch-run process from previous attempt has not exited")
+ continue
+ }
go sch.lockContainer(logger, ctr.UUID)
unalloc[it]--
case arvados.ContainerStateLocked:
// a higher-priority container on the
// same instance type. Don't let this
// one sneak in ahead of it.
- } else if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ } else if sch.pool.KillContainer(ctr.UUID, "about to start") {
logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
} else if sch.pool.StartContainer(it, ctr) {
// Success.
}
if sis.allowCreateCall.After(time.Now()) {
return nil, RateLimitError{sis.allowCreateCall}
- } else {
- sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls)
}
-
+ sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls)
ak := sis.driver.AuthorizedKeys
if authKey != nil {
ak = append([]ssh.PublicKey{authKey}, ak...)
defer sis.mtx.RUnlock()
if sis.allowInstancesCall.After(time.Now()) {
return nil, RateLimitError{sis.allowInstancesCall}
- } else {
- sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls)
}
+ sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls)
var r []cloud.Instance
for _, ss := range sis.servers {
r = append(r, ss.Instance())
if running {
fmt.Fprintf(stderr, "%s: container is running\n", uuid)
return 1
- } else {
- fmt.Fprintf(stderr, "%s: container is not running\n", uuid)
- return 0
}
+ fmt.Fprintf(stderr, "%s: container is not running\n", uuid)
+ return 0
}
if command == "true" {
return 0
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder)
+ make_builder, arvados_jobs_image)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
from .perf import Perf
sum_res_pars = ("outdirMin", "outdirMax")
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
- submit_runner_ram=0, name=None, merged_map=None):
+ submit_runner_ram=0, name=None, merged_map=None,
+ submit_runner_image=None):
packed = packed_workflow(arvRunner, tool, merged_map)
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
+ wf_runner_resources = None
+
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ wf_runner_resources = h
+ found = True
+ break
+ if not found:
+ wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+ hints.append(wf_runner_resources)
+
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
if submit_runner_ram:
- hints = main.get("hints", [])
- found = False
- for h in hints:
- if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
- h["ramMin"] = submit_runner_ram
- found = True
- break
- if not found:
- hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
- "ramMin": submit_runner_ram})
- main["hints"] = hints
+ wf_runner_resources["ramMin"] = submit_runner_ram
+
+ main["hints"] = hints
body = {
"workflow": {
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
- merged_map=merged_map),
+ merged_map=merged_map,
+ submit_runner_image=runtimeContext.submit_runner_image),
"success")
self.apply_reqs(job_order, tool)
#
# SPDX-License-Identifier: Apache-2.0
+fpm_depends+=(nodejs)
+
case "$TARGET" in
- debian8)
- fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
- ;;
debian9 | ubuntu1604)
fpm_depends+=(libcurl3-gnutls)
;;
"$graph": [
{
"class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
"id": "#main",
"inputs": [],
"outputs": [],
}
],
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
- open("tests/wf/expect_packed.cwl").read())
+ open("tests/wf/expect_upload_packed.cwl").read().rstrip())
@stubs
def test_create(self, stubs):
stubs.capture_stdout, sys.stderr, api_client=stubs.api)
toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
- expect_workflow = StripYAMLComments(open(toolfile).read())
+ expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
body = {
"workflow": {
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "baseCommand": "cat",
+ "class": "CommandLineTool",
+ "id": "#submit_tool.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt"
+ },
+ "id": "#submit_tool.cwl/x",
+ "inputBinding": {
+ "position": 1
+ },
+ "type": "File"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/x",
+ "source": "#main/x"
+ }
+ ],
+ "out": [],
+ "run": "#submit_tool.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
if mBlkRe.MatchString(tok) {
return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
- } else {
- return tok
}
+ return tok
})
}
var DefaultConfigFile = func() string {
if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
return path
- } else {
- return "/etc/arvados/config.yml"
}
+ return "/etc/arvados/config.yml"
}()
type Config struct {
return nil
}
-// Index returns an unsorted list of blocks at the given mount point.
+// IndexMount returns an unsorted list of blocks at the given mount point.
func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
}
e.HttpStatusCode,
e.HttpStatusMessage,
e.ServerAddress)
- } else {
- return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
- e.HttpStatusCode,
- e.HttpStatusMessage,
- e.ServerAddress)
}
+ return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
+ e.HttpStatusCode,
+ e.HttpStatusMessage,
+ e.ServerAddress)
}
// StringBool tests whether s is suggestive of true. It returns true
value, found = c.DiscoveryDoc[parameter]
if found {
return value, nil
- } else {
- return value, ErrInvalidArgument
}
+ return value, ErrInvalidArgument
}
func (ac *ArvadosClient) httpClient() *http.Client {
func (rs RootSorter) getWeight(hash string, uuid string) string {
if len(uuid) == 27 {
return Md5String(hash + uuid[12:])
- } else {
- // Only useful for testing, a set of one service root, etc.
- return Md5String(hash + uuid)
}
+ // Only useful for testing, a set of one service root, etc.
+ return Md5String(hash + uuid)
}
func (rs RootSorter) GetSortedRoots() []string {
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
- next_server := 0
+ nextServer := 0
// The number of active writers
active := 0
for retriesRemaining > 0 {
retriesRemaining -= 1
- next_server = 0
+ nextServer = 0
retryServers = []string{}
for replicasTodo > 0 {
for active*replicasPerThread < replicasTodo {
// Start some upload requests
- if next_server < len(sv) {
- DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[next_server])
- go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, reqid)
- next_server += 1
+ if nextServer < len(sv) {
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
+ go this.uploadToKeepServer(sv[nextServer], hash, getReader(), upload_status, expectedLength, reqid)
+ nextServer += 1
active += 1
} else {
if active == 0 && retriesRemaining == 0 {
}
msg = msg[:len(msg)-2]
return locator, replicasDone, InsufficientReplicasError(errors.New(msg))
- } else {
- break
}
+ break
}
}
DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v",
act_as_system_user
def create_api_client_auth(supplied_token=nil)
+ supplied_token = Rails.configuration.Users["AnonymousUserToken"]
- # If token is supplied, see if it exists
- if supplied_token
- api_client_auth = ApiClientAuthorization.
- where(api_token: supplied_token).
- first
- if !api_client_auth
- # fall through to create a token
- else
- raise "Token exists, aborting!"
+ if supplied_token.nil? or supplied_token.empty?
+ puts "Users.AnonymousUserToken is empty. Destroying tokens that belong to anonymous."
+ # Token is empty. Destroy any anonymous tokens.
+ ApiClientAuthorization.where(user: anonymous_user).destroy_all
+ return nil
+ end
+
+ attr = {user: anonymous_user,
+ api_client_id: 0,
+ scopes: ['GET /']}
+
+ secret = supplied_token
+
+ if supplied_token[0..2] == 'v2/'
+ _, token_uuid, secret, optional = supplied_token.split('/')
+ if token_uuid[0..4] != Rails.configuration.ClusterID
+ # Belongs to a different cluster.
+ puts supplied_token
+ return nil
end
+ attr[:uuid] = token_uuid
end
- api_client_auth = ApiClientAuthorization.
- new(user: anonymous_user,
- api_client_id: 0,
- expires_at: Time.now + 100.years,
- scopes: ['GET /'],
- api_token: supplied_token)
- api_client_auth.save!
- api_client_auth.reload
+ attr[:api_token] = secret
+
+ api_client_auth = ApiClientAuthorization.where(attr).first
+ if !api_client_auth
+ api_client_auth = ApiClientAuthorization.create!(attr)
+ end
api_client_auth
end
end
# print it to the console
-puts api_client_auth.api_token
+if api_client_auth
+ puts "v2/#{api_client_auth.uuid}/#{api_client_auth.api_token}"
+end
id: ex_string_def
default: hello-testing-123
outputs: []
+
+workflow_with_wrr:
+ uuid: zzzzz-7fd4e-validwithinput3
+ owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
+ name: Workflow with WorkflowRunnerResources
+ description: this workflow has WorkflowRunnerResources
+ created_at: <%= 1.minute.ago.to_s(:db) %>
+ definition: |
+ cwlVersion: v1.0
+ class: CommandLineTool
+ hints:
+ - class: http://arvados.org/cwl#WorkflowRunnerResources
+ acrContainerImage: arvados/jobs:2.0.4
+ ramMin: 1234
+ coresMin: 2
+ keep_cache: 678
+ baseCommand:
+ - echo
+ inputs:
+ - type: string
+ id: ex_string
+ - type: string
+ id: ex_string_def
+ default: hello-testing-123
+ outputs: []
from future.utils import viewitems
from future.utils import itervalues
from builtins import dict
-import logging
-import re
-import time
-import llfuse
-import arvados
import apiclient
+import arvados
+import errno
import functools
+import llfuse
+import logging
+import re
+import sys
import threading
-from apiclient import errors as apiclient_errors
-import errno
import time
+from apiclient import errors as apiclient_errors
from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
from .fresh import FreshBase, convertTime, use_counter, check_update
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
- import sys
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
import errno
import os
import subprocess
+import sys
import time
walkpath = ""
}
- resp := s3.ListResp{
- Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
- Prefix: params.prefix,
- Delimiter: params.delimiter,
- Marker: params.marker,
- MaxKeys: params.maxKeys,
+ type commonPrefix struct {
+ Prefix string
+ }
+ type listResp struct {
+ XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+ s3.ListResp
+ // s3.ListResp marshals an empty tag when
+ // CommonPrefixes is nil, which confuses some clients.
+ // Fix by using this nested struct instead.
+ CommonPrefixes []commonPrefix
+ }
+ resp := listResp{
+ ListResp: s3.ListResp{
+ Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+ Prefix: params.prefix,
+ Delimiter: params.delimiter,
+ Marker: params.marker,
+ MaxKeys: params.maxKeys,
+ },
}
commonPrefixes := map[string]bool{}
err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
return
}
if params.delimiter != "" {
+ resp.CommonPrefixes = make([]commonPrefix, 0, len(commonPrefixes))
for prefix := range commonPrefixes {
- resp.CommonPrefixes = append(resp.CommonPrefixes, prefix)
- sort.Strings(resp.CommonPrefixes)
+ resp.CommonPrefixes = append(resp.CommonPrefixes, commonPrefix{prefix})
}
+ sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
}
- wrappedResp := struct {
- XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
- s3.ListResp
- }{"", resp}
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
- if err := xml.NewEncoder(w).Encode(wrappedResp); err != nil {
+ if err := xml.NewEncoder(w).Encode(resp); err != nil {
ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")
}
}
}
}
+// If there are no CommonPrefixes entries, the CommonPrefixes XML tag
+// should not appear at all.
+func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
+}
+
func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
logins = arv.virtual_machine.logins(:uuid => vm_uuid)[:items]
logins = [] if logins.nil?
- logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:virtual_machine_uuid] != vm_uuid }
+ logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:virtual_machine_uuid] != vm_uuid }
# No system users
uid_min = 1000
logins.each do |l|
keys[l[:username]] = Array.new() if not keys.has_key?(l[:username])
key = l[:public_key]
- # Handle putty-style ssh public keys
- key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
- key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
- key.gsub!(/\n/,'')
- key.strip
-
- keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ if !key.nil?
+ # Handle putty-style ssh public keys
+ key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ key.gsub!(/\n/,'')
+ key.strip
+
+ keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ end
end
seen = Hash.new()
fi
}
+listusers() {
+ docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+}
+
wait_for_arvbox() {
FF=/tmp/arvbox-fifo-$$
mkfifo $FF
wait_for_arvbox
echo "The Arvados source code is checked out at: $ARVADOS_ROOT"
echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem"
+ if [[ "$(listusers)" =~ ^\{\} ]] ; then
+ echo "No users defined, use 'arvbox adduser' to add user logins"
+ else
+ echo "Use 'arvbox listusers' to see user logins"
+ fi
else
echo "Unknown configuration '$CONFIG'"
fi
;;
listusers)
- exec docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+ listusers
;;
*)
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
"http://localhost:${services[controller]}": {}
- RailsAPI:
- InternalURLs:
- "http://localhost:${services[api]}/": {}
PostgreSQL:
ConnectionPool: 32 # max concurrent connections per arvados server daemon
Connection:
projects / arvados.git / commitdiff