if hint[:keep_cache]
keep_cache = hint[:keep_cache]
end
+ if hint[:acrContainerImage]
+ attrs['container_image'] = hint[:acrContainerImage]
+ end
end
end
end
assert_text 'script version'
assert_no_selector 'a', text: 'Run this pipeline'
else
- within first('tr[data-kind="arvados#workflow"]') do
+ within 'tr[data-kind="arvados#workflow"]', text: "Workflow with default input specifications" do
click_link 'Show'
end
assert_text "This container request was created from the workflow"
assert_match /Provide a value for .* then click the \"Run\" button to start the workflow/, page.text
end
+
+ test "create workflow with WorkflowRunnerResources" do
+ visit page_with_token('active', '/workflows/zzzzz-7fd4e-validwithinput3')
+
+ find('a,button', text: 'Run this workflow').click
+
+ # Choose project for the container_request being created
+ within('.modal-dialog') do
+ find('.selectable', text: 'A Project').click
+ find('button', text: 'Choose').click
+ end
+ click_link 'Advanced'
+ click_link("API response")
+ assert_text('"container_image": "arvados/jobs:2.0.4"')
+ assert_text('"vcpus": 2')
+ assert_text('"ram": 1293942784')
+ assert_text('"--collection-cache-size=678"')
+
+ end
end
$ git clone https://github.com/arvados/arvados.git
$ cd arvados/tools/arvbox/bin
$ ./arvbox start localdemo
+$ ./arvbox adduser demouser demo@example.com
</pre>
+You can now log in as @demouser@ using the password you selected.
+
h2. Requirements
* Linux 3.x+ and Docker 1.9+
build <config> build arvbox Docker image
reboot <config> stop, build arvbox Docker image, run
rebuild <config> build arvbox Docker image, no layer cache
+checkpoint create database backup
+restore restore checkpoint
+hotreset reset database and restart API without restarting container
reset delete arvbox arvados data (be careful!)
destroy delete all arvbox code and data (be careful!)
log <service> tail log of specified service
sv <start|stop|restart> <service>
change state of service inside arvbox
clone <from> <to> clone dev arvbox
+adduser <username> <email>
+ add a user login
+removeuser <username>
+ remove user login
+listusers list user logins
</pre>
h2. Install root certificate
--- /dev/null
+---
+layout: default
+navsection: sdk
+navmenu: Python
+title: Arvados CWL Runner
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+The Arvados FUSE driver is a Python utility that allows you to see the Keep service as a normal filesystem, so that data can be accessed using standard tools. This driver requires the Python SDK installed in order to access Arvados services.
+
+h2. Installation
+
+If you are logged in to a managed Arvados VM, the @arv-mount@ utility should already be installed.
+
+To use the FUSE driver elsewhere, you can install from a distribution package, or PyPI.
+
+h2. Option 1: Install from distribution packages
+
+First, "add the appropriate package repository for your distribution":{{ site.baseurl }}/install/packages.html
+
+{% assign arvados_component = 'python3-arvados-cwl-runner' %}
+
+{% include 'install_packages' %}
+
+h2. Option 2: Install with pip
+
+Run @pip install arvados-cwl-runner@ in an appropriate installation environment, such as a virtualenv.
+
+Note:
+
+The SDK uses @pycurl@ which depends on the @libcurl@ C library. To build the module you may have to first install additional packages. On Debian 9 this is:
+
+<pre>
+$ apt-get install git build-essential python-dev libcurl4-openssl-dev libssl1.0-dev python-llfuse
+</pre>
+
+For Python 3 this is:
+
+<pre>
+$ apt-get install git build-essential python3-dev libcurl4-openssl-dev libssl1.0-dev python3-llfuse
+</pre>
+
+h3. Check Docker access
+
+In order to pull and upload Docker images, @arvados-cwl-runner@ requires access to Docker. You do not need Docker if the Docker images you intend to use are already available in Arvados.
+
+You can determine if you have access to Docker by running @docker version@:
+
+<notextile>
+<pre><code>~$ <span class="userinput">docker version</span>
+Client:
+ Version: 1.9.1
+ API version: 1.21
+ Go version: go1.4.2
+ Git commit: a34a1d5
+ Built: Fri Nov 20 12:59:02 UTC 2015
+ OS/Arch: linux/amd64
+
+Server:
+ Version: 1.9.1
+ API version: 1.21
+ Go version: go1.4.2
+ Git commit: a34a1d5
+ Built: Fri Nov 20 12:59:02 UTC 2015
+ OS/Arch: linux/amd64
+</code></pre>
+</notextile>
+
+If this returns an error, contact the sysadmin of your cluster for assistance.
+
+h3. Usage
+
+Please refer to the "Accessing Keep from GNU/Linux":{{site.baseurl}}/user/tutorials/tutorial-keep-mount-gnu-linux.html tutorial for more information.
}),
})
if (!resp.ok) {
- document.getElementById('error').innerHTML = 'authentication failed (default accounts are user/user, admin/admin)'
+ document.getElementById('error').innerHTML = '<p>Authentication failed.</p><p>The "test login" users are defined in Clusters.[ClusterID].Login.Test.Users section of config.yml</p><p>If you are using arvbox, use "arvbox adduser" to add users.</p>'
return
}
var redir = document.getElementById('return_to').value
type TokenProvider func(context.Context) ([]string, error)
func PassthroughTokenProvider(ctx context.Context) ([]string, error) {
- if incoming, ok := auth.FromContext(ctx); !ok {
+ incoming, ok := auth.FromContext(ctx)
+ if !ok {
return nil, errors.New("no token provided")
- } else {
- return incoming.Tokens, nil
}
+ return incoming.Tokens, nil
}
type Conn struct {
u.User = nil
u.Host = ""
return u.String()
- } else {
- return location
}
+ return location
}
func (conn *Conn) CollectionCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Collection, error) {
return err
}
return w.Close()
- } else {
- // Dispatched via crunch-dispatch-slurm. Look up
- // apiserver's node record corresponding to
- // $SLURMD_NODENAME.
- hostname := os.Getenv("SLURMD_NODENAME")
- if hostname == "" {
- hostname, _ = os.Hostname()
- }
- _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) {
- // The "info" field has admin-only info when
- // obtained with a privileged token, and
- // should not be logged.
- node, ok := resp.(map[string]interface{})
- if ok {
- delete(node, "info")
- }
- })
- return err
}
+ // Dispatched via crunch-dispatch-slurm. Look up
+ // apiserver's node record corresponding to
+ // $SLURMD_NODENAME.
+ hostname := os.Getenv("SLURMD_NODENAME")
+ if hostname == "" {
+ hostname, _ = os.Hostname()
+ }
+ _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) {
+ // The "info" field has admin-only info when
+ // obtained with a privileged token, and
+ // should not be logged.
+ node, ok := resp.(map[string]interface{})
+ if ok {
+ delete(node, "info")
+ }
+ })
+ return err
}
func (runner *ContainerRunner) logAPIResponse(label, path string, params map[string]interface{}, munge func(interface{})) (logged bool, err error) {
return runner.token, nil
}
-// UpdateContainerComplete updates the container record state on API
+// UpdateContainerFinal updates the container record state on API
// server to "Complete" or "Cancelled"
func (runner *ContainerRunner) UpdateContainerFinal() error {
update := arvadosclient.Dict{}
"golang.org/x/crypto/ssh"
)
-// Map of available cloud drivers.
+// Drivers is a map of available cloud drivers.
// Clusters.*.Containers.CloudVMs.Driver configuration values
// correspond to keys in this map.
var Drivers = map[string]cloud.Driver{
func boolLabelValue(v bool) string {
if v {
return "1"
- } else {
- return "0"
}
+ return "0"
}
overquota = sorted[i:]
break tryrun
}
+ if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ logger.Info("not locking: crunch-run process from previous attempt has not exited")
+ continue
+ }
go sch.lockContainer(logger, ctr.UUID)
unalloc[it]--
case arvados.ContainerStateLocked:
// a higher-priority container on the
// same instance type. Don't let this
// one sneak in ahead of it.
- } else if sch.pool.KillContainer(ctr.UUID, "about to lock") {
+ } else if sch.pool.KillContainer(ctr.UUID, "about to start") {
logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
} else if sch.pool.StartContainer(it, ctr) {
// Success.
}
if sis.allowCreateCall.After(time.Now()) {
return nil, RateLimitError{sis.allowCreateCall}
- } else {
- sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls)
}
-
+ sis.allowCreateCall = time.Now().Add(sis.driver.MinTimeBetweenCreateCalls)
ak := sis.driver.AuthorizedKeys
if authKey != nil {
ak = append([]ssh.PublicKey{authKey}, ak...)
defer sis.mtx.RUnlock()
if sis.allowInstancesCall.After(time.Now()) {
return nil, RateLimitError{sis.allowInstancesCall}
- } else {
- sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls)
}
+ sis.allowInstancesCall = time.Now().Add(sis.driver.MinTimeBetweenInstancesCalls)
var r []cloud.Instance
for _, ss := range sis.servers {
r = append(r, ss.Instance())
if running {
fmt.Fprintf(stderr, "%s: container is running\n", uuid)
return 1
- } else {
- fmt.Fprintf(stderr, "%s: container is not running\n", uuid)
- return 0
}
+ fmt.Fprintf(stderr, "%s: container is not running\n", uuid)
+ return 0
}
if command == "true" {
return 0
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
MiB. Default 256 MiB. Will be added on to the RAM request
when determining node size to request.
jsonldPredicate: "http://arvados.org/cwl#RuntimeConstraints/keep_cache"
+ acrContainerImage:
+ type: string?
+ doc: |
+ The container image containing the correct version of
+ arvados-cwl-runner to use when invoking the workflow on
+ Arvados.
- name: ClusterTarget
type: record
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
- make_builder)
+ make_builder, arvados_jobs_image)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
+from ._version import __version__
from .perf import Perf
sum_res_pars = ("outdirMin", "outdirMax")
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
- submit_runner_ram=0, name=None, merged_map=None):
+ submit_runner_ram=0, name=None, merged_map=None,
+ submit_runner_image=None):
packed = packed_workflow(arvRunner, tool, merged_map)
upload_dependencies(arvRunner, name, tool.doc_loader,
packed, tool.tool["id"], False)
+ wf_runner_resources = None
+
+ hints = main.get("hints", [])
+ found = False
+ for h in hints:
+ if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
+ wf_runner_resources = h
+ found = True
+ break
+ if not found:
+ wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
+ hints.append(wf_runner_resources)
+
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner, submit_runner_image or "arvados/jobs:"+__version__)
+
if submit_runner_ram:
- hints = main.get("hints", [])
- found = False
- for h in hints:
- if h["class"] == "http://arvados.org/cwl#WorkflowRunnerResources":
- h["ramMin"] = submit_runner_ram
- found = True
- break
- if not found:
- hints.append({"class": "http://arvados.org/cwl#WorkflowRunnerResources",
- "ramMin": submit_runner_ram})
- main["hints"] = hints
+ wf_runner_resources["ramMin"] = submit_runner_ram
+
+ main["hints"] = hints
body = {
"workflow": {
uuid=existing_uuid,
submit_runner_ram=runtimeContext.submit_runner_ram,
name=runtimeContext.name,
- merged_map=merged_map),
+ merged_map=merged_map,
+ submit_runner_image=runtimeContext.submit_runner_image),
"success")
self.apply_reqs(job_order, tool)
#
# SPDX-License-Identifier: Apache-2.0
+fpm_depends+=(nodejs)
+
case "$TARGET" in
- debian8)
- fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
- ;;
debian9 | ubuntu1604)
fpm_depends+=(libcurl3-gnutls)
;;
"$graph": [
{
"class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
"id": "#main",
"inputs": [],
"outputs": [],
}
],
"cwlVersion": "v1.0"
-}
\ No newline at end of file
+}
class TestCreateWorkflow(unittest.TestCase):
existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
expect_workflow = StripYAMLComments(
- open("tests/wf/expect_packed.cwl").read())
+ open("tests/wf/expect_upload_packed.cwl").read().rstrip())
@stubs
def test_create(self, stubs):
stubs.capture_stdout, sys.stderr, api_client=stubs.api)
toolfile = "tests/collection_per_tool/collection_per_tool_packed.cwl"
- expect_workflow = StripYAMLComments(open(toolfile).read())
+ expect_workflow = StripYAMLComments(open(toolfile).read().rstrip())
body = {
"workflow": {
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "baseCommand": "cat",
+ "class": "CommandLineTool",
+ "id": "#submit_tool.cwl",
+ "inputs": [
+ {
+ "default": {
+ "class": "File",
+ "location": "keep:5d373e7629203ce39e7c22af98a0f881+52/blub.txt"
+ },
+ "id": "#submit_tool.cwl/x",
+ "inputBinding": {
+ "position": 1
+ },
+ "type": "File"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "DockerRequirement",
+ "dockerPull": "debian:8",
+ "http://arvados.org/cwl#dockerCollectionPDH": "999999999999999999999999999999d4+99"
+ }
+ ]
+ },
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "steps": [
+ {
+ "id": "#main/step1",
+ "in": [
+ {
+ "id": "#main/step1/x",
+ "source": "#main/x"
+ }
+ ],
+ "out": [],
+ "run": "#submit_tool.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.0"
+}
return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
if mBlkRe.MatchString(tok) {
return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
- } else {
- return tok
}
+ return tok
})
}
var DefaultConfigFile = func() string {
if path := os.Getenv("ARVADOS_CONFIG"); path != "" {
return path
- } else {
- return "/etc/arvados/config.yml"
}
+ return "/etc/arvados/config.yml"
}()
type Config struct {
return nil
}
-// Index returns an unsorted list of blocks at the given mount point.
+// IndexMount returns an unsorted list of blocks at the given mount point.
func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
}
e.HttpStatusCode,
e.HttpStatusMessage,
e.ServerAddress)
- } else {
- return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
- e.HttpStatusCode,
- e.HttpStatusMessage,
- e.ServerAddress)
}
+ return fmt.Sprintf("arvados API server error: %d: %s returned by %s",
+ e.HttpStatusCode,
+ e.HttpStatusMessage,
+ e.ServerAddress)
}
// StringBool tests whether s is suggestive of true. It returns true
value, found = c.DiscoveryDoc[parameter]
if found {
return value, nil
- } else {
- return value, ErrInvalidArgument
}
+ return value, ErrInvalidArgument
}
func (ac *ArvadosClient) httpClient() *http.Client {
func (rs RootSorter) getWeight(hash string, uuid string) string {
if len(uuid) == 27 {
return Md5String(hash + uuid[12:])
- } else {
- // Only useful for testing, a set of one service root, etc.
- return Md5String(hash + uuid)
}
+ // Only useful for testing, a set of one service root, etc.
+ return Md5String(hash + uuid)
}
func (rs RootSorter) GetSortedRoots() []string {
sv := NewRootSorter(this.WritableLocalRoots(), hash).GetSortedRoots()
// The next server to try contacting
- next_server := 0
+ nextServer := 0
// The number of active writers
active := 0
for retriesRemaining > 0 {
retriesRemaining -= 1
- next_server = 0
+ nextServer = 0
retryServers = []string{}
for replicasTodo > 0 {
for active*replicasPerThread < replicasTodo {
// Start some upload requests
- if next_server < len(sv) {
- DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[next_server])
- go this.uploadToKeepServer(sv[next_server], hash, getReader(), upload_status, expectedLength, reqid)
- next_server += 1
+ if nextServer < len(sv) {
+ DebugPrintf("DEBUG: [%s] Begin upload %s to %s", reqid, hash, sv[nextServer])
+ go this.uploadToKeepServer(sv[nextServer], hash, getReader(), upload_status, expectedLength, reqid)
+ nextServer += 1
active += 1
} else {
if active == 0 && retriesRemaining == 0 {
}
msg = msg[:len(msg)-2]
return locator, replicasDone, InsufficientReplicasError(errors.New(msg))
- } else {
- break
}
+ break
}
}
DebugPrintf("DEBUG: [%s] Replicas remaining to write: %v active uploads: %v",
id: ex_string_def
default: hello-testing-123
outputs: []
+
+workflow_with_wrr:
+ uuid: zzzzz-7fd4e-validwithinput3
+ owner_uuid: zzzzz-j7d0g-zhxawtyetzwc5f0
+ name: Workflow with WorkflowRunnerResources
+ description: this workflow has WorkflowRunnerResources
+ created_at: <%= 1.minute.ago.to_s(:db) %>
+ definition: |
+ cwlVersion: v1.0
+ class: CommandLineTool
+ hints:
+ - class: http://arvados.org/cwl#WorkflowRunnerResources
+ acrContainerImage: arvados/jobs:2.0.4
+ ramMin: 1234
+ coresMin: 2
+ keep_cache: 678
+ baseCommand:
+ - echo
+ inputs:
+ - type: string
+ id: ex_string
+ - type: string
+ id: ex_string_def
+ default: hello-testing-123
+ outputs: []
from future.utils import viewitems
from future.utils import itervalues
from builtins import dict
-import logging
-import re
-import time
-import llfuse
-import arvados
import apiclient
+import arvados
+import errno
import functools
+import llfuse
+import logging
+import re
+import sys
import threading
-from apiclient import errors as apiclient_errors
-import errno
import time
+from apiclient import errors as apiclient_errors
from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
from .fresh import FreshBase, convertTime, use_counter, check_update
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
- import sys
e = self.inodes.add_entry(CollectionDirectory(
self.inode, self.inodes, self.api, self.num_retries, k))
import errno
import os
import subprocess
+import sys
import time
walkpath = ""
}
- resp := s3.ListResp{
- Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
- Prefix: params.prefix,
- Delimiter: params.delimiter,
- Marker: params.marker,
- MaxKeys: params.maxKeys,
+ type commonPrefix struct {
+ Prefix string
+ }
+ type listResp struct {
+ XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+ s3.ListResp
+ // s3.ListResp marshals an empty tag when
+ // CommonPrefixes is nil, which confuses some clients.
+ // Fix by using this nested struct instead.
+ CommonPrefixes []commonPrefix
+ }
+ resp := listResp{
+ ListResp: s3.ListResp{
+ Name: strings.SplitN(r.URL.Path[1:], "/", 2)[0],
+ Prefix: params.prefix,
+ Delimiter: params.delimiter,
+ Marker: params.marker,
+ MaxKeys: params.maxKeys,
+ },
}
commonPrefixes := map[string]bool{}
err := walkFS(fs, strings.TrimSuffix(bucketdir+"/"+walkpath, "/"), true, func(path string, fi os.FileInfo) error {
return
}
if params.delimiter != "" {
+ resp.CommonPrefixes = make([]commonPrefix, 0, len(commonPrefixes))
for prefix := range commonPrefixes {
- resp.CommonPrefixes = append(resp.CommonPrefixes, prefix)
- sort.Strings(resp.CommonPrefixes)
+ resp.CommonPrefixes = append(resp.CommonPrefixes, commonPrefix{prefix})
}
+ sort.Slice(resp.CommonPrefixes, func(i, j int) bool { return resp.CommonPrefixes[i].Prefix < resp.CommonPrefixes[j].Prefix })
}
- wrappedResp := struct {
- XMLName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
- s3.ListResp
- }{"", resp}
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
- if err := xml.NewEncoder(w).Encode(wrappedResp); err != nil {
+ if err := xml.NewEncoder(w).Encode(resp); err != nil {
ctxlog.FromContext(r.Context()).WithError(err).Error("error writing xml response")
}
}
}
}
+// If there are no CommonPrefixes entries, the CommonPrefixes XML tag
+// should not appear at all.
+func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+
+ req, err := http.NewRequest("GET", stage.collbucket.URL("/"), nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = "prefix=asdfasdfasdf&delimiter=/"
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Not(check.Matches), `(?ms).*CommonPrefixes.*`)
+}
+
func (s *IntegrationSuite) TestS3CollectionList(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
logins = arv.virtual_machine.logins(:uuid => vm_uuid)[:items]
logins = [] if logins.nil?
- logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:public_key].nil? or l[:virtual_machine_uuid] != vm_uuid }
+ logins = logins.reject { |l| l[:username].nil? or l[:hostname].nil? or l[:virtual_machine_uuid] != vm_uuid }
# No system users
uid_min = 1000
logins.each do |l|
keys[l[:username]] = Array.new() if not keys.has_key?(l[:username])
key = l[:public_key]
- # Handle putty-style ssh public keys
- key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
- key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
- key.gsub!(/\n/,'')
- key.strip
-
- keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ if !key.nil?
+ # Handle putty-style ssh public keys
+ key.sub!(/^(Comment: "r[^\n]*\n)(.*)$/m,'ssh-rsa \2 \1')
+ key.sub!(/^(Comment: "d[^\n]*\n)(.*)$/m,'ssh-dss \2 \1')
+ key.gsub!(/\n/,'')
+ key.strip
+
+ keys[l[:username]].push(key) if not keys[l[:username]].include?(key)
+ end
end
seen = Hash.new()
fi
}
+listusers() {
+ docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml $(getclusterid) list
+}
+
wait_for_arvbox() {
FF=/tmp/arvbox-fifo-$$
mkfifo $FF
git -C "$COMPOSER_ROOT" pull
fi
if ! test -d "$WORKBENCH2_ROOT" ; then
- git clone https://github.com/arvados/arvados-workbench2.git "$WORKBENCH2_ROOT"
+ git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
fi
if [[ "$CONFIG" = test ]] ; then
wait_for_arvbox
echo "The Arvados source code is checked out at: $ARVADOS_ROOT"
echo "The Arvados testing root certificate is $VAR_DATA/root-cert.pem"
+ if [[ "$(listusers)" =~ ^\{\} ]] ; then
+ echo "No users defined, use 'arvbox adduser' to add user logins"
+ else
+ echo "Use 'arvbox listusers' to see user logins"
+ fi
else
echo "Unknown configuration '$CONFIG'"
fi
EOF
;;
+ adduser)
+ docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml.override $(getclusterid) add $@
+ docker exec $ARVBOX_CONTAINER sv restart controller
+ ;;
+
+ removeuser)
+ docker exec -ti $ARVBOX_CONTAINER /usr/local/lib/arvbox/edit_users.py /var/lib/arvados/cluster_config.yml.override $(getclusterid) remove $@
+ docker exec $ARVBOX_CONTAINER sv restart controller
+ ;;
+
+ listusers)
+ listusers
+ ;;
+
*)
echo "Arvados-in-a-box https://doc.arvados.org/install/arvbox.html"
echo
echo "sv <start|stop|restart> <service> "
echo " change state of service inside arvbox"
echo "clone <from> <to> clone dev arvbox"
+ echo "adduser <username> <email>"
+ echo " add a user login"
+ echo "removeuser <username>"
+ echo " remove user login"
+ echo "listusers list user logins"
;;
esac
libgnutls28-dev python3-dev vim cadaver cython gnupg dirmngr \
libsecret-1-dev r-base r-cran-testthat libxml2-dev pandoc \
python3-setuptools python3-pip openjdk-8-jdk bsdmainutils net-tools \
- ruby2.3 ruby-dev bundler && \
+ ruby2.3 ruby-dev bundler shellinabox && \
apt-get clean
ENV RUBYVERSION_MINOR 2.3
keep-setup.sh common.sh createusers.sh \
logger runsu.sh waitforpostgres.sh \
yml_override.py api-setup.sh \
- go-setup.sh devenv.sh cluster-config.sh \
+ go-setup.sh devenv.sh cluster-config.sh edit_users.py \
/usr/local/lib/arvbox/
ADD runit /etc/runit
ARG workbench2_version=master
RUN cd /usr/src && \
- git clone --no-checkout https://github.com/arvados/arvados.git && \
+ git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
git -C arvados pull && \
git clone --no-checkout https://github.com/arvados/composer.git && \
git -C composer checkout ${composer_version} && \
git -C composer pull && \
- git clone --no-checkout https://github.com/arvados/arvados-workbench2.git workbench2 && \
+ git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
git -C workbench2 checkout ${workbench2_version} && \
git -C workbench2 pull && \
chown -R 1000:1000 /usr/src
InternalURLs:
"http://localhost:${services[keep-web]}/": {}
ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
- InternalURLs:
- "http://localhost:${services[keep-web]}/": {}
Composer:
ExternalURL: "https://$localip:${services[composer]}"
Controller:
ExternalURL: "https://$localip:${services[controller-ssl]}"
InternalURLs:
"http://localhost:${services[controller]}": {}
- RailsAPI:
- InternalURLs:
- "http://localhost:${services[api]}/": {}
+ WebShell:
+ InternalURLs: {}
+ ExternalURL: "https://$localip:${services[webshell-ssl]}"
PostgreSQL:
ConnectionPool: 32 # max concurrent connections per arvados server daemon
Connection:
Login:
Test:
Enable: true
- Users:
- admin:
- Email: admin@example.com
- Password: admin
- user:
- Email: user@example.com
- Password: user
Users:
NewUsersAreActive: true
- AutoAdminUserWithEmail: admin@example.com
+ AutoAdminFirstUser: true
AutoSetupNewUsers: true
AutoSetupNewUsersWithVmUUID: $vm_uuid
AutoSetupNewUsersWithRepository: true
cp /var/lib/arvados/cluster_config.yml /etc/arvados/config.yml
+chmod og-rw \
+ /var/lib/arvados/cluster_config.yml.override \
+ /var/lib/arvados/cluster_config.yml \
+ /etc/arvados/config.yml \
+ /var/lib/arvados/api_secret_token \
+ /var/lib/arvados/blob_signing_key \
+ /var/lib/arvados/management_token \
+ /var/lib/arvados/system_root_token \
+ /var/lib/arvados/api_database_pw \
+ /var/lib/arvados/workbench_secret_token \
+ /var/lib/arvados/superuser_token \
+
mkdir -p /var/lib/arvados/run_tests
cat >/var/lib/arvados/run_tests/config.yml <<EOF
Clusters:
[doc]=8001
[websockets]=8005
[websockets-ssl]=8002
+ [webshell]=4201
+ [webshell-ssl]=4202
)
if test "$(id arvbox -u 2>/dev/null)" = 0 ; then
--- /dev/null
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+import ruamel.yaml
+import sys
+import getpass
+import os
+
+def print_help():
+ print("%s <path/to/config.yaml> <clusterid> add <username> <email> [pass]" % (sys.argv[0]))
+ print("%s <path/to/config.yaml> <clusterid> remove <username>" % (" " * len(sys.argv[0])))
+ print("%s <path/to/config.yaml> <clusterid> list" % (" " * len(sys.argv[0])))
+ exit()
+
+if len(sys.argv) < 4:
+ print_help()
+
+fn = sys.argv[1]
+cl = sys.argv[2]
+op = sys.argv[3]
+
+if op == "remove" and len(sys.argv) < 5:
+ print_help()
+if op == "add" and len(sys.argv) < 6:
+ print_help()
+
+if op in ("add", "remove"):
+ user = sys.argv[4]
+
+if not os.path.exists(fn):
+ open(fn, "w").close()
+
+with open(fn, "r") as f:
+ conf = ruamel.yaml.round_trip_load(f)
+
+if not conf:
+ conf = {}
+
+conf["Clusters"] = conf.get("Clusters", {})
+conf["Clusters"][cl] = conf["Clusters"].get(cl, {})
+conf["Clusters"][cl]["Login"] = conf["Clusters"][cl].get("Login", {})
+conf["Clusters"][cl]["Login"]["Test"] = conf["Clusters"][cl]["Login"].get("Test", {})
+conf["Clusters"][cl]["Login"]["Test"]["Users"] = conf["Clusters"][cl]["Login"]["Test"].get("Users", {})
+
+users_obj = conf["Clusters"][cl]["Login"]["Test"]["Users"]
+
+if op == "add":
+ email = sys.argv[5]
+ if len(sys.argv) == 7:
+ p = sys.argv[6]
+ else:
+ p = getpass.getpass("Password for %s: " % user)
+
+ users_obj[user] = {
+ "Email": email,
+ "Password": p
+ }
+ print("Added %s" % user)
+elif op == "remove":
+ del users_obj[user]
+ print("Removed %s" % user)
+elif op == "list":
+ print(ruamel.yaml.round_trip_dump(users_obj))
+else:
+ print("Operations are 'add', 'remove' and 'list'")
+
+with open(fn, "w") as f:
+ f.write(ruamel.yaml.round_trip_dump(conf))
}
}
+
+upstream arvados-webshell {
+ server localhost:${services[webshell]};
+}
+server {
+ listen ${services[webshell-ssl]} ssl;
+ server_name arvados-webshell;
+
+ proxy_connect_timeout 90s;
+ proxy_read_timeout 300s;
+
+ ssl on;
+ ssl_certificate "${server_cert}";
+ ssl_certificate_key "${server_cert_key}";
+
+ location / {
+ if (\$request_method = 'OPTIONS') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ add_header 'Access-Control-Max-Age' 1728000;
+ add_header 'Content-Type' 'text/plain charset=UTF-8';
+ add_header 'Content-Length' 0;
+ return 204;
+ }
+ if (\$request_method = 'POST') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+ if (\$request_method = 'GET') {
+ add_header 'Access-Control-Allow-Origin' '*';
+ add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+ add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+ }
+
+ proxy_ssl_session_reuse off;
+ proxy_read_timeout 90;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Host \$http_host;
+ proxy_set_header X-Real-IP \$remote_addr;
+ proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+ proxy_pass http://arvados-webshell;
+ }
+}
}
EOF
--- /dev/null
+/usr/local/lib/arvbox/logger
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -ex
+
+. /usr/local/lib/arvbox/common.sh
+
+/usr/local/lib/arvbox/runsu.sh $0-service
+
+cat > /etc/pam.d/shellinabox <<EOF
+# This example is a stock debian "login" file with pam_arvados
+# replacing pam_unix. It can be installed as /etc/pam.d/shellinabox .
+
+auth optional pam_faildelay.so delay=3000000
+auth [success=ok new_authtok_reqd=ok ignore=ignore user_unknown=bad default=die] pam_securetty.so
+auth requisite pam_nologin.so
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so close
+session required pam_env.so readenv=1
+session required pam_env.so readenv=1 envfile=/etc/default/locale
+
+auth [success=1 default=ignore] /usr/local/lib/pam_arvados.so $localip:${services[controller-ssl]} $localip
+auth requisite pam_deny.so
+auth required pam_permit.so
+
+auth optional pam_group.so
+session required pam_limits.so
+session optional pam_lastlog.so
+session optional pam_motd.so motd=/run/motd.dynamic
+session optional pam_motd.so
+session optional pam_mail.so standard
+
+@include common-account
+@include common-session
+@include common-password
+
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so open
+EOF
+
+exec shellinaboxd --verbose --port ${services[webshell]} --user arvbox --group arvbox \
+ --disable-ssl --no-beep --service=/:AUTH:HOME:SHELL
\ No newline at end of file
--- /dev/null
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+flock /var/lib/gopath/gopath.lock go build -buildmode=c-shared -o ${GOPATH}/bin/pam_arvados.so git.arvados.org/arvados.git/lib/pam
+install $GOPATH/bin/pam_arvados.so /usr/local/lib
\ No newline at end of file
projects / arvados.git / commitdiff