h2. Resource
-Deprecated. A pipeline instance is a collection of jobs managed by @aravdos-run-pipeline-instance@.
+Deprecated. A pipeline instance is a collection of jobs managed by @arvados-run-pipeline-instance@.
Each PipelineInstance has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html:
Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
-The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a shell account and password on the controller host itself. This can be convenient for a single-user or test cluster.
+The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. User accounts can have @/dev/false@ as the shell in order to allow the user to log into Arvados but not log into a shell on the controller host.
PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
print("Finished downloading %s" % filename)
{% endcodeblock %}
-h2. Copy files from a collection a new collection
+h2. Copy files from a collection to a new collection
{% codeblock as python %}
import arvados.collection
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
-h2. Copy files from a collection another collection
+h2. Copy files from a collection to another collection
{% codeblock as python %}
import arvados.collection
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: e21c963c05fb35bd8be15e24f72045e9c1cec668
+git hash: f39d44c1bdb2f82ec8f22ade874ca70544531289
{% endcomment %}
1. Vscode: On the left sidebar, choose `Explorer` 
1. Select `Clone Repository` and enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training), then click `Open`
1. If asked `Would you like to open the cloned repository?` choose `Open`
-1. Vscode: Click on the `Terminal` menu
+1. Go to Arvados Workbench
+ 1. Workbench: In the user menu, select `Current token`
+ 1. Workbench: Click on `Copy to Clipboard`.
+ 1. Workbench: You should see a notification `Token copied to clipboard`.
+ 1. Go to Vscode
+ 1. Vscode: Click on the `Terminal` menu
1. Vscode: Click `Run Task…`
1. Vscode: Select `Configure Arvados`
- 1. Go to Arvados Workbench
- 1. Workbench: In the user menu, select `Current token`
- 1. Workbench: Copy the text following `ARVADOS_API_HOST=` to the end of the line
- 1. Vscode: Paste the string at the `Value for ARVADOS_API_HOST` prompt
- 1. Workbench: Copy the text following `ARVADOS_API_TOKEN=` to the end of the line
- 1. Vscode: Paste the string at the `Value for ARVADOS_API_TOKEN` prompt
- 1. This will create files called `API_HOST` and `API_TOKEN`
+ 1. Vscode: Paste text into the `Current API_TOKEN and API_HOST from Workbench` prompt
+ 1. Vscode: This will create files called `API_HOST` and `API_TOKEN`
## 3. Register & run a workflow
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
ret.FirstName = strings.Join(names[0:len(names)-1], " ")
ret.LastName = names[len(names)-1]
- } else {
+ } else if len(names) > 0 {
ret.FirstName = names[0]
}
ret.Email, _ = claims[ctrl.EmailClaim].(string)
Log: cr.CrunchLog,
}
os.Unsetenv("GatewayAuthSecret")
- err = cr.gateway.Start()
- if err != nil {
- log.Printf("error starting gateway server: %s", err)
- return 1
+ if cr.gateway.Address != "" {
+ err = cr.gateway.Start()
+ if err != nil {
+ log.Printf("error starting gateway server: %s", err)
+ return 1
+ }
}
parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
s.cluster = &arvados.Cluster{
ManagementToken: "test-management-token",
Containers: arvados.ContainersConfig{
- DispatchPrivateKey: string(dispatchprivraw),
- StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
+ CrunchRunCommand: "crunch-run",
+ CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
+ DispatchPrivateKey: string(dispatchprivraw),
+ StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
CloudVMs: arvados.CloudVMsConfig{
Driver: "test",
SyncInterval: arvados.Duration(10 * time.Millisecond),
stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
stubvm.ExecuteContainer = executeContainer
stubvm.CrashRunningContainer = finishContainer
+ stubvm.ExtraCrunchRunArgs = "'--foo' '--extra='\\''args'\\'''"
switch n % 7 {
case 0:
stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
ArvMountDeadlockRate float64
ExecuteContainer func(arvados.Container) int
CrashRunningContainer func(arvados.Container)
+ ExtraCrunchRunArgs string // extra args expected after "crunch-run --detach --stdin-env "
sis *StubInstanceSet
id cloud.InstanceID
fmt.Fprint(stderr, "crunch-run: command not found\n")
return 1
}
- if strings.HasPrefix(command, "crunch-run --detach --stdin-env ") {
+ if strings.HasPrefix(command, "crunch-run --detach --stdin-env "+svm.ExtraCrunchRunArgs) {
var stdinKV map[string]string
err := json.Unmarshal(stdinData, &stdinKV)
if err != nil {
systemRootToken: cluster.SystemRootToken,
installPublicKey: installPublicKey,
tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix,
+ runnerCmdDefault: cluster.Containers.CrunchRunCommand,
+ runnerArgs: cluster.Containers.CrunchRunArgumentsList,
stop: make(chan bool),
}
wp.registerMetrics(reg)
systemRootToken string
installPublicKey ssh.PublicKey
tagKeyPrefix string
+ runnerCmdDefault string // crunch-run command to use if not deploying a binary
+ runnerArgs []string // extra args passed to crunch-run
// private state
subscribers map[<-chan struct{}]chan<- struct{}
if wp.runnerData != nil {
return nil
} else if wp.runnerSource == "" {
- wp.runnerCmd = "crunch-run"
+ wp.runnerCmd = wp.runnerCmdDefault
wp.runnerData = []byte{}
return nil
}
newExecutor := func(cloud.Instance) Executor {
return &stubExecutor{
response: map[string]stubResp{
- "crunch-run --list": {},
- "true": {},
+ "crunch-run-custom --list": {},
+ "true": {},
},
}
}
SyncInterval: arvados.Duration(time.Millisecond * 10),
TagKeyPrefix: "testprefix:",
},
+ CrunchRunCommand: "crunch-run-custom",
},
InstanceTypes: arvados.InstanceTypeMap{
type1.Name: type1,
"encoding/json"
"fmt"
"net"
+ "strings"
"syscall"
"time"
executor Executor
envJSON json.RawMessage
runnerCmd string
+ runnerArgs []string
remoteUser string
timeoutTERM time.Duration
timeoutSignal time.Duration
executor: wkr.executor,
envJSON: envJSON,
runnerCmd: wkr.wp.runnerCmd,
+ runnerArgs: wkr.wp.runnerArgs,
remoteUser: wkr.instance.RemoteUser(),
timeoutTERM: wkr.wp.timeoutTERM,
timeoutSignal: wkr.wp.timeoutSignal,
// assume the remote process _might_ have started, at least until it
// probes the worker and finds otherwise.
func (rr *remoteRunner) Start() {
- cmd := rr.runnerCmd + " --detach --stdin-env '" + rr.uuid + "'"
+ cmd := rr.runnerCmd + " --detach --stdin-env"
+ for _, arg := range rr.runnerArgs {
+ cmd += " '" + strings.Replace(arg, "'", "'\\''", -1) + "'"
+ }
+ cmd += " '" + rr.uuid + "'"
if rr.remoteUser != "root" {
cmd = "sudo " + cmd
}
timeoutBooting: bootTimeout,
timeoutProbe: probeTimeout,
exited: map[string]time.Time{},
+ runnerCmdDefault: "crunch-run",
+ runnerArgs: []string{"--args=not used with --list"},
runnerCmd: "crunch-run",
runnerData: trial.deployRunner,
runnerMD5: md5.Sum(trial.deployRunner),
projects / arvados.git / commitdiff