end
end
+task :import_vscode_training do
+ Dir.chdir("user") do
+ rm_rf "arvados-vscode-cwl-training"
+ `git clone https://github.com/arvados/arvados-vscode-cwl-training`
+ githash = `git --git-dir arvados-vscode-cwl-training/.git log -n1 --format=%H HEAD`
+ File.open("cwl/arvados-vscode-training.html.md.liquid", "w") do |fn|
+ File.open("arvados-vscode-cwl-training/README.md", "r") do |rd|
+ fn.write(<<-EOF
+---
+layout: default
+navsection: userguide
+title: "Developing CWL Workflows with VSCode"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+
+Imported from https://github.com/arvados/arvados-vscode-cwl-training
+git hash: #{githash}
+{% endcomment %}
+
+EOF
+ )
+ fn.write(rd.read())
+ end
+ end
+ rm_rf "arvados-vscode-cwl-training"
+ end
+end
+
task :clean do
rm_rf "sdk/python/arvados"
rm_rf "sdk/R"
- user/topics/collection-versioning.html.textile.liquid
- user/topics/storage-classes.html.textile.liquid
- Data Analysis with Workflows:
+ - user/cwl/arvados-vscode-training.html.md.liquid
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
h2. Resource
-Deprecated. A pipeline instance is a collection of jobs managed by @aravdos-run-pipeline-instance@.
+Deprecated. A pipeline instance is a collection of jobs managed by @arvados-run-pipeline-instance@.
Each PipelineInstance has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html:
Clusters are identified by a five-digit alphanumeric id (numbers and lowercase letters). There are 36 ^5^ = 60466176 possible cluster identifiers.
-* For automated tests purposes, use "z****"
+* For automated test purposes, use "z****"
* For experimental/local-only/private clusters that won't ever be visible on the public Internet, use "x****"
-* For long-lived clusters, we recommend reserving a cluster id. Contact "info@curii.com":mailto:info@curii.com
+* For long-lived clusters, we recommend reserving a cluster id. Contact "info@curii.com":mailto:info@curii.com for more information.
Cluster identifiers are mapped API server hosts one of two ways:
h2(#clusterid). Arvados Cluster ID
-Each Arvados installation should have a cluster identifier, which is a unique 5-character lowercase alphanumeric string. Here is one way to make a random 5-character string:
+Each Arvados installation is identified by a cluster identifier, which is a unique 5-character lowercase alphanumeric string. There are 36 5 = 60466176 possible cluster identifiers.
+
+* For automated test purposes, use “z****”
+* For experimental/local-only/private clusters that won’t ever be visible on the public Internet, use “x****”
+* For long-lived clusters, we recommend reserving a cluster id. Contact "info@curii.com":mailto:info@curii.com for more information.
+
+Here is one way to make a random 5-character string:
<notextile>
<pre><code>~$ <span class="userinput">tr -dc 0-9a-z </dev/urandom | head -c5; echo</span>
Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
-The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a shell account and password on the controller host itself. This can be convenient for a single-user or test cluster.
+The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. User accounts can have @/dev/false@ as the shell in order to allow the user to log into Arvados but not log into a shell on the controller host.
PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
print("Finished downloading %s" % filename)
{% endcodeblock %}
-h2. Copy files from a collection a new collection
+h2. Copy files from a collection to a new collection
{% codeblock as python %}
import arvados.collection
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
-h2. Copy files from a collection another collection
+h2. Copy files from a collection to another collection
{% codeblock as python %}
import arvados.collection
--- /dev/null
+---
+layout: default
+navsection: userguide
+title: "Developing CWL Workflows with VSCode"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+
+Imported from https://github.com/arvados/arvados-vscode-cwl-training
+git hash: f39d44c1bdb2f82ec8f22ade874ca70544531289
+
+{% endcomment %}
+
+These lessons give step by step instructions for using Visual Studio
+Code (abbreviated "vscode") to develop CWL workflows on Arvados.
+
+1. Set up SSH
+1. Install vscode and necessary extensions, then use vscode to connect to an Arvados shell node for development
+1. Register a workflow, run it on workbench, and view the log
+1. Upload input, run a workflow on it, and view the output
+1. Register a workflow with default inputs
+1. Run a workflow without registering it
+
+## 1. SSH Setup
+
+1. (Windows only) Install Git for Windows [https://git-scm.com/download/win](https://git-scm.com/download/win)
+ 1. Choose "64-bit Git for Windows Setup". It does not require admin privileges to install.
+ 1. Hit "Next" a bunch of times to accept the defaults
+ 1. The most important things is that "install git bash" and "install OpenSSH" are enabled (this is the default).
+ 1. At the end of the installation, you can launch tick a box to git bash directly.
+ 1. Open "Git Bash" (installed in the "Git" folder of the start menu)
+1. (All operating systems) Starting from bash shell (on MacOS or Linux you will open "Terminal")
+ 1. Shell: Run `ssh-keygen`
+ 1. Hit enter to save to a default location
+ 1. You can choose to protect the key with a password, or just hit enter for no password.
+ 1. Shell: Look for a message like `Your public key has been saved
+ in /c/Users/MyUsername/.ssh/id_rsa.pub` (Windows git bash
+ example, on MacOS or Linux this will probably start with `/Users` or `/home`)
+ 1. Shell: Run `cat /c/Users/MyUsername/.ssh/id_rsa.pub`
+ 1. Shell: Use the pointer to highlight and copy the lines starting
+ with `ssh-rsa …` up to the next blank line. Right click and
+ select "Copy"
+1. Open Arvados workbench 2. If necessary, go to the user menu and
+ select "Go to Workbench 2"
+ 1. Workbench: Go to `SSH keys` in the user menu
+ 1. Workbench:Click `+Add new ssh key`
+ 1. Workbench: Paste the key into `Public key` and enter something for `name`
+ 1. Workbench: Go to `Virtual Machines` in the user menu
+ 1. Workbench: Highlight and copy the value in in the `Command line` column.
+1. At the git bash command line
+ 1. Shell: paste the `ssh shell…` command line you got from workbench.
+ 1. Shell: type "yes" if it asks `Are you sure you want to continue connecting`.
+ 1. Note: it can take up to two minutes for the SSH key to be copied to
+ the shell node. If you get "Permission denied" the first time, wait 60
+ seconds and try again.
+ 1. Shell: You should now be logged into the Arvados shell node.
+ 1. Shell: Log out by typing `exit`
+
+## 2. VSCode setup
+
+1. Install [Visual Studio Code](https://code.visualstudio.com/) and start it up
+1. Vscode: On the left sidebar, select `Extensions` 
+ 1. In `Search Extensions in Marketplace` enter "remote development".
+ 1. Choose and install the "Remote Development" extension pack from Microsoft
+1. Vscode: On the left sidebar, choose `Remote Explorer` 
+ 1. At the top of the Remote Explorer panel choose `SSH targets` 
+ 1. Click `Add New` 
+ 1. Enter the `ssh shell…` command line you used in the previous section, step 1.4.1
+ 1. If it asks you `Select SSH configuration file to update` choose the first one in the list.
+ 1. Right click the newly added ssh target in the list and select “connect to host in current window`
+ 1. If it asks `Select platform of the remote host` select `Linux`.
+1. Vscode: On the left sidebar, go back to `Extensions` 
+ 1. Search for "benten", then look for `CWL (Rabix/Benten)` and click `Install`
+ 1. On the information page for `CWL (Rabix/Benten)`
+ 1. If you see a warning `Install the extension on 'SSH: ...' to enable` then click the button `Install in SSH: ...`
+ 1. You should now see a message `Extension is enabled on 'SSH: ...' and disabled locally.`
+1. Vscode: On the left sidebar, choose `Explorer` 
+ 1. Select `Clone Repository` and enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training), then click `Open`
+ 1. If asked `Would you like to open the cloned repository?` choose `Open`
+1. Go to Arvados Workbench
+ 1. Workbench: In the user menu, select `Current token`
+ 1. Workbench: Click on `Copy to Clipboard`.
+ 1. Workbench: You should see a notification `Token copied to clipboard`.
+ 1. Go to Vscode
+ 1. Vscode: Click on the `Terminal` menu
+ 1. Vscode: Click `Run Task…`
+ 1. Vscode: Select `Configure Arvados`
+ 1. Vscode: Paste text into the `Current API_TOKEN and API_HOST from Workbench` prompt
+ 1. Vscode: This will create files called `API_HOST` and `API_TOKEN`
+
+## 3. Register & run a workflow
+
+1. Vscode: Click on the `lesson1/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
+ 1. This will create a file called `WORKFLOW_UUID`
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 1"
+ 1. You should arrive at the panel for the new project
+1. Workbench: With `Lesson 1` selected
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 1` from the list and click `Next`
+ 1. Enter a name for this run like `First training run`
+ 1. Enter a message (under `#main/message`) like "Hello world"
+ 1. Click `Run process`
+ 1. This should take you to a panel showing the workflow run status
+1. Workbench: workflow run status panel
+ 1. Wait for the badge in the upper right to say `Completed`
+ 1. In the lower panel, double click on the `echo` workflow step
+ 1. This will take you to the status panel for the `echo` step
+ 1. Click on the three vertical dots in the top-right corner next to `Completed`
+ 1. Choose `Log`
+ 1. This will take you to the log viewer panel
+ 1. Under `Event Type` choose `stdout`
+ 1. You should see your message
+
+## 4. Working with input and output files
+
+1. Vscode: Click on the `lesson2/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
+1. Go to your desktop
+ 1. Right click on the desktop, select `New > Text Document`
+ 1. Name the file `message`
+ 1. Enter a message like "Hello earth" and save
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 2"
+ 1. You should arrive at the panel for the new project
+1. Arvados workbench: With `Lesson 2` project selected
+ 1. Click on +NEW and select `New collection`
+ 1. For Collection Name enter "my message"
+ 1. Drag and drop `message.txt` into the browser
+ 1. Click `Create a collection`
+ 1. The file should be uploaded and then you will be on the collection page
+1. Workbench: Select the `Lesson 2` project
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 2` from the list and click `Next`
+ 1. Enter a name for this run like "Second training run"
+ 1. Click on `#main/message`
+ 1. A selection dialog box will appear
+ 1. Navigate to the collection you created in step (4.4.4) and choose `message.txt`
+ 1. Click `Run process`
+ 1. This should take you to a panel showing the workflow run status
+1. Workbench: workflow run status panel
+ 1. Wait for the process to complete
+ 1. Click on the dot menu
+ 1. Choose `Outputs`
+ 1. Right click on `reverse.txt`
+ 1. Click on `Open in new tab`
+ 1. The results should be visible in a new browser tab.
+
+## 5. Register a workflow with default inputs
+
+The default value for the `message` parameter will taken from the `lesson3/defaults.yaml` file
+
+1. Vscode: Click on the `lesson3/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 3"
+ 1. You should arrive at the panel for the new project
+1. Workbench: With `Lesson 3` selected
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 3` from the list and click `Next`
+ 1. Enter a name for this run like "Third training run"
+ 1. The `#main/message` parameter will be pre-filled with your default value. You can choose to change it or use the default.
+ 1. Click `Run process`
+ 1. This should take you to the status page for this workflow
+ 1. The greeting will appear in the `Log` of the `echo` task, which
+ can be found the same way as described earlier in section 3.
+
+## 6. Run a workflow without registering it
+
+The `message` parameter will be taken from the file `lesson4/main-input.yaml`. This is useful during development.
+
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 4"
+ 1. You should arrive at the panel for the new project
+ 1. Click on `Additional info` in the upper right to expand the `info` panel
+ 1. Under `Project UUID` click the `Copy to clipboard` button
+1. Vscode: Select the file `lesson4/main.cwl`
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Set Arvados project UUID`
+ 1. Paste the project UUID from workbench at the prompt
+1. Vscode: Select the file `lesson4/main.cwl`
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Run CWL workflow on Arvados`
+1. Vscode: In the bottom panel select the `Terminal` tab
+ 1. In the upper right corner of the Terminal tab select `Task - Run CWL Workflow` from the drop-down
+ 1. Look for logging text like `submitted container_request zzzzz-xvhdp-0123456789abcde`
+ 1. Highlight and copy the workflow identifier (this the string containing `-xvhdp-` in the middle)
+ 1. The results of this run will appear in the terminal when the run completes.
+1. Workbench: Paste the workflow identifier into the search box
+ 1. This will take you to the status page for this workflow
+
+
+## Notes
+
+If you need to change something about the environment of the user on
+the remote host (for example, the user has been added to a new unix
+group) you need to restart the vscode server that runs on the remote
+host. Do this in vscode:
+
+ctrl+shift+p: `Remote-SSH: Kill VS Code Server on Host`
+
+This is because the vscode server remains running on the remote host
+even after you disconnect, so exiting/restarting vscode on the desktop
+has no effect.
---
layout: default
navsection: userguide
-title: "Developing workflows with CWL"
+title: "CWL Resources"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
} else {
san += fmt.Sprintf(",DNS:%s", super.ListenHost)
}
- if hostname, err := os.Hostname(); err != nil {
+ hostname, err := os.Hostname()
+ if err != nil {
return fmt.Errorf("hostname: %w", err)
- } else {
- san += ",DNS:" + hostname
}
+ san += ",DNS:" + hostname
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
dbname: ""
SAMPLE: ""
API:
+ # Limits for how long a client token created by regular users can be valid,
+ # and also is used as a default expiration policy when no expiration date is
+ # specified.
+ # Default value zero means token expirations don't get clamped and no
+ # default expiration is set.
+ MaxTokenLifetime: 0s
+
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Note: You must separately configure the upstream web server or
TrustAllContent: false
# Cache parameters for WebDAV content serving:
- # * TTL: Maximum time to cache manifests and permission checks.
- # * UUIDTTL: Maximum time to cache collection state.
- # * MaxBlockEntries: Maximum number of block cache entries.
- # * MaxCollectionEntries: Maximum number of collection cache entries.
- # * MaxCollectionBytes: Approximate memory limit for collection cache.
- # * MaxPermissionEntries: Maximum number of permission cache entries.
- # * MaxUUIDEntries: Maximum number of UUID cache entries.
WebDAVCache:
+ # Time to cache manifests, permission checks, and sessions.
TTL: 300s
+
+ # Time to cache collection state.
UUIDTTL: 5s
- MaxBlockEntries: 4
+
+ # Block cache entries. Each block consumes up to 64 MiB RAM.
+ MaxBlockEntries: 4
+
+ # Collection cache entries.
MaxCollectionEntries: 1000
- MaxCollectionBytes: 100000000
+
+ # Approximate memory limit (in bytes) for collection cache.
+ MaxCollectionBytes: 100000000
+
+ # Permission cache entries.
MaxPermissionEntries: 1000
- MaxUUIDEntries: 1000
+
+ # UUID cache entries.
+ MaxUUIDEntries: 1000
+
+ # Persistent sessions.
+ MaxSessions: 100
Login:
# One of the following mechanisms (SSO, Google, PAM, LDAP, or
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
"API.MaxKeepBlobBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
+ "API.MaxTokenLifetime": false,
"API.RequestTimeout": true,
"API.SendTimeout": true,
"API.WebsocketClientEventQueue": false,
type ExportSuite struct{}
func (s *ExportSuite) TestExport(c *check.C) {
- confdata := strings.Replace(string(DefaultYAML), "SAMPLE", "testkey", -1)
+ confdata := strings.Replace(string(DefaultYAML), "SAMPLE", "12345", -1)
cfg, err := testLoader(c, confdata, nil).Load()
c.Assert(err, check.IsNil)
cluster, err := cfg.GetCluster("xxxxx")
dbname: ""
SAMPLE: ""
API:
+ # Limits for how long a client token created by regular users can be valid,
+ # and also is used as a default expiration policy when no expiration date is
+ # specified.
+ # Default value zero means token expirations don't get clamped and no
+ # default expiration is set.
+ MaxTokenLifetime: 0s
+
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Note: You must separately configure the upstream web server or
TrustAllContent: false
# Cache parameters for WebDAV content serving:
- # * TTL: Maximum time to cache manifests and permission checks.
- # * UUIDTTL: Maximum time to cache collection state.
- # * MaxBlockEntries: Maximum number of block cache entries.
- # * MaxCollectionEntries: Maximum number of collection cache entries.
- # * MaxCollectionBytes: Approximate memory limit for collection cache.
- # * MaxPermissionEntries: Maximum number of permission cache entries.
- # * MaxUUIDEntries: Maximum number of UUID cache entries.
WebDAVCache:
+ # Time to cache manifests, permission checks, and sessions.
TTL: 300s
+
+ # Time to cache collection state.
UUIDTTL: 5s
- MaxBlockEntries: 4
+
+ # Block cache entries. Each block consumes up to 64 MiB RAM.
+ MaxBlockEntries: 4
+
+ # Collection cache entries.
MaxCollectionEntries: 1000
- MaxCollectionBytes: 100000000
+
+ # Approximate memory limit (in bytes) for collection cache.
+ MaxCollectionBytes: 100000000
+
+ # Permission cache entries.
MaxPermissionEntries: 1000
- MaxUUIDEntries: 1000
+
+ # UUID cache entries.
+ MaxUUIDEntries: 1000
+
+ # Persistent sessions.
+ MaxSessions: 100
Login:
# One of the following mechanisms (SSO, Google, PAM, LDAP, or
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
// Check for known mistakes
for id, cc := range cfg.Clusters {
+ for remote, _ := range cc.RemoteClusters {
+ if remote == "*" || remote == "SAMPLE" {
+ continue
+ }
+ err = ldr.checkClusterID(fmt.Sprintf("Clusters.%s.RemoteClusters.%s", id, remote), remote, true)
+ if err != nil {
+ return nil, err
+ }
+ }
for _, err = range []error{
+ ldr.checkClusterID(fmt.Sprintf("Clusters.%s", id), id, false),
+ ldr.checkClusterID(fmt.Sprintf("Clusters.%s.Login.LoginCluster", id), cc.Login.LoginCluster, true),
ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken),
ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken),
ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey),
return &cfg, nil
}
+var acceptableClusterIDRe = regexp.MustCompile(`^[a-z0-9]{5}$`)
+
+func (ldr *Loader) checkClusterID(label, clusterID string, emptyStringOk bool) error {
+ if emptyStringOk && clusterID == "" {
+ return nil
+ } else if !acceptableClusterIDRe.MatchString(clusterID) {
+ return fmt.Errorf("%s: cluster ID should be 5 alphanumeric characters", label)
+ }
+ return nil
+}
+
var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`)
var acceptableTokenLength = 32
c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
}
+func (s *LoadSuite) TestBadClusterIDs(c *check.C) {
+ for _, data := range []string{`
+Clusters:
+ 123456:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+`, `
+Clusters:
+ 12345:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ RemoteClusters:
+ Zzzzz:
+ Host: Zzzzz.arvadosapi.com
+ Proxy: true
+`, `
+Clusters:
+ abcde:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Login:
+ LoginCluster: zz-zz
+`,
+ } {
+ c.Log(data)
+ v, err := testLoader(c, data, nil).Load()
+ if v != nil {
+ c.Logf("%#v", v.Clusters)
+ }
+ c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 alphanumeric characters.*`)
+ }
+}
+
func (s *LoadSuite) TestBadType(c *check.C) {
for _, data := range []string{`
Clusters:
arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
s.testHandler.Cluster.ClusterID = "zzzzz"
s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
+ s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
+ s.testHandler.Cluster.Collections.BlobSigningTTL = arvados.Duration(336 * time.Hour) // For some reason, this was set to 0h
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
// Runtime token must match zzzzz cluster
c.Check(cr.RuntimeToken, check.Matches, "v2/zzzzz-gj3su-.*")
+
// RuntimeToken must be different than the Original Token we originally did the request with.
c.Check(cr.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
+
+ // Runtime token should not have an expiration based on API.MaxTokenLifetime
+ req2 := httptest.NewRequest("GET", "/arvados/v1/api_client_authorizations/current", nil)
+ req2.Header.Set("Authorization", "Bearer "+cr.RuntimeToken)
+ req2.Header.Set("Content-type", "application/json")
+ resp = s.testRequest(req2).Result()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ var aca arvados.APIClientAuthorization
+ c.Check(json.NewDecoder(resp.Body).Decode(&aca), check.IsNil)
+ c.Check(aca.ExpiresAt, check.NotNil) // Time.Now()+BlobSigningTTL
+ t, _ := time.Parse(time.RFC3339Nano, aca.ExpiresAt)
+ c.Check(t.After(time.Now().Add(s.testHandler.Cluster.API.MaxTokenLifetime.Duration())), check.Equals, true)
+ c.Check(t.Before(time.Now().Add(s.testHandler.Cluster.Collections.BlobSigningTTL.Duration())), check.Equals, true)
}
func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
ret.FirstName = strings.Join(names[0:len(names)-1], " ")
ret.LastName = names[len(names)-1]
- } else {
+ } else if len(names) > 0 {
ret.FirstName = names[0]
}
ret.Email, _ = claims[ctrl.EmailClaim].(string)
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
if c.User() == "_" {
return nil, nil
- } else {
- return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
},
PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
if c.User() == "_" {
"pubkey-fp": ssh.FingerprintSHA256(pubKey),
},
}, nil
- } else {
- return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
},
}
pvt, err := rsa.GenerateKey(rand.Reader, 2048)
Log: cr.CrunchLog,
}
os.Unsetenv("GatewayAuthSecret")
- err = cr.gateway.Start()
- if err != nil {
- log.Printf("error starting gateway server: %s", err)
- return 1
+ if cr.gateway.Address != "" {
+ err = cr.gateway.Start()
+ if err != nil {
+ log.Printf("error starting gateway server: %s", err)
+ return 1
+ }
}
parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
s.cluster = &arvados.Cluster{
ManagementToken: "test-management-token",
Containers: arvados.ContainersConfig{
- DispatchPrivateKey: string(dispatchprivraw),
- StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
+ CrunchRunCommand: "crunch-run",
+ CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
+ DispatchPrivateKey: string(dispatchprivraw),
+ StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
CloudVMs: arvados.CloudVMsConfig{
Driver: "test",
SyncInterval: arvados.Duration(10 * time.Millisecond),
stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
stubvm.ExecuteContainer = executeContainer
stubvm.CrashRunningContainer = finishContainer
+ stubvm.ExtraCrunchRunArgs = "'--foo' '--extra='\\''args'\\'''"
switch n % 7 {
case 0:
stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
ArvMountDeadlockRate float64
ExecuteContainer func(arvados.Container) int
CrashRunningContainer func(arvados.Container)
+ ExtraCrunchRunArgs string // extra args expected after "crunch-run --detach --stdin-env "
sis *StubInstanceSet
id cloud.InstanceID
fmt.Fprint(stderr, "crunch-run: command not found\n")
return 1
}
- if strings.HasPrefix(command, "crunch-run --detach --stdin-env ") {
+ if strings.HasPrefix(command, "crunch-run --detach --stdin-env "+svm.ExtraCrunchRunArgs) {
var stdinKV map[string]string
err := json.Unmarshal(stdinData, &stdinKV)
if err != nil {
systemRootToken: cluster.SystemRootToken,
installPublicKey: installPublicKey,
tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix,
+ runnerCmdDefault: cluster.Containers.CrunchRunCommand,
+ runnerArgs: cluster.Containers.CrunchRunArgumentsList,
stop: make(chan bool),
}
wp.registerMetrics(reg)
systemRootToken string
installPublicKey ssh.PublicKey
tagKeyPrefix string
+ runnerCmdDefault string // crunch-run command to use if not deploying a binary
+ runnerArgs []string // extra args passed to crunch-run
// private state
subscribers map[<-chan struct{}]chan<- struct{}
if wp.runnerData != nil {
return nil
} else if wp.runnerSource == "" {
- wp.runnerCmd = "crunch-run"
+ wp.runnerCmd = wp.runnerCmdDefault
wp.runnerData = []byte{}
return nil
}
newExecutor := func(cloud.Instance) Executor {
return &stubExecutor{
response: map[string]stubResp{
- "crunch-run --list": {},
- "true": {},
+ "crunch-run-custom --list": {},
+ "true": {},
},
}
}
SyncInterval: arvados.Duration(time.Millisecond * 10),
TagKeyPrefix: "testprefix:",
},
+ CrunchRunCommand: "crunch-run-custom",
},
InstanceTypes: arvados.InstanceTypeMap{
type1.Name: type1,
"encoding/json"
"fmt"
"net"
+ "strings"
"syscall"
"time"
executor Executor
envJSON json.RawMessage
runnerCmd string
+ runnerArgs []string
remoteUser string
timeoutTERM time.Duration
timeoutSignal time.Duration
executor: wkr.executor,
envJSON: envJSON,
runnerCmd: wkr.wp.runnerCmd,
+ runnerArgs: wkr.wp.runnerArgs,
remoteUser: wkr.instance.RemoteUser(),
timeoutTERM: wkr.wp.timeoutTERM,
timeoutSignal: wkr.wp.timeoutSignal,
// assume the remote process _might_ have started, at least until it
// probes the worker and finds otherwise.
func (rr *remoteRunner) Start() {
- cmd := rr.runnerCmd + " --detach --stdin-env '" + rr.uuid + "'"
+ cmd := rr.runnerCmd + " --detach --stdin-env"
+ for _, arg := range rr.runnerArgs {
+ cmd += " '" + strings.Replace(arg, "'", "'\\''", -1) + "'"
+ }
+ cmd += " '" + rr.uuid + "'"
if rr.remoteUser != "root" {
cmd = "sudo " + cmd
}
timeoutBooting: bootTimeout,
timeoutProbe: probeTimeout,
exited: map[string]time.Time{},
+ runnerCmdDefault: "crunch-run",
+ runnerArgs: []string{"--args=not used with --list"},
runnerCmd: "crunch-run",
runnerData: trial.deployRunner,
runnerMD5: md5.Sum(trial.deployRunner),
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20201121085451',
- 'schema-salad==7.0.20200612160654',
+ 'cwltool==3.0.20210124104916',
+ 'schema-salad==7.0.20210124093443',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
if c.APIHost == "" {
if c.loadedFromEnv {
return errors.New("ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set")
- } else {
- return errors.New("arvados.Client cannot perform request: APIHost is not set")
}
+ return errors.New("arvados.Client cannot perform request: APIHost is not set")
}
urlString := c.apiURL(path)
urlValues, err := anythingToValues(params)
MaxCollectionBytes int64
MaxPermissionEntries int
MaxUUIDEntries int
+ MaxSessions int
}
type Cluster struct {
MaxKeepBlobBuffers int
MaxRequestAmplification int
MaxRequestSize int
+ MaxTokenLifetime Duration
RequestTimeout Duration
SendTimeout Duration
WebsocketClientEventQueue int
// path is "", flush all dirs/streams; otherwise, flush only
// the specified dir/stream.
Flush(path string, shortBlocks bool) error
+
+ // Estimate current memory usage.
+ MemorySize() int64
}
type inode interface {
sync.Locker
RLock()
RUnlock()
+ MemorySize() int64
}
type fileinfo struct {
return nil, ErrNotADirectory
}
+func (*nullnode) MemorySize() int64 {
+ // Types that embed nullnode should report their own size, but
+ // if they don't, we at least report a non-zero size to ensure
+ // a large tree doesn't get reported as 0 bytes.
+ return 64
+}
+
type treenode struct {
fs FileSystem
parent inode
return nil
}
+func (n *treenode) MemorySize() (size int64) {
+ n.RLock()
+ defer n.RUnlock()
+ for _, inode := range n.inodes {
+ size += inode.MemorySize()
+ }
+ return
+}
+
type fileSystem struct {
root inode
fsBackend
return ErrInvalidOperation
}
+func (fs *fileSystem) MemorySize() int64 {
+ return fs.root.MemorySize()
+}
+
// rlookup (recursive lookup) returns the inode for the file/directory
// with the given name (which may contain "/" separators). If no such
// file/directory exists, the returned node is nil.
// Total data bytes in all files.
Size() int64
-
- // Memory consumed by buffered file data.
- memorySize() int64
}
type collectionFileSystem struct {
return dn.flush(context.TODO(), names, flushOpts{sync: false, shortBlocks: shortBlocks})
}
-func (fs *collectionFileSystem) memorySize() int64 {
+func (fs *collectionFileSystem) MemorySize() int64 {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
- return fs.fileSystem.root.(*dirnode).memorySize()
+ return fs.fileSystem.root.(*dirnode).MemorySize()
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
}
// caller must have write lock.
-func (dn *dirnode) memorySize() (size int64) {
+func (dn *dirnode) MemorySize() (size int64) {
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
node.Lock()
defer node.Unlock()
switch node := node.(type) {
case *dirnode:
- size += node.memorySize()
+ size += node.MemorySize()
case *filenode:
for _, seg := range node.segments {
switch seg := seg.(type) {
fs.Flush("", true)
}
- size := fs.memorySize()
+ size := fs.MemorySize()
if !c.Check(size <= 1<<24, check.Equals, true) {
- c.Logf("at dir%d fs.memorySize()=%d", i, size)
+ c.Logf("at dir%d fs.MemorySize()=%d", i, size)
return
}
}
c.Assert(err, check.IsNil)
}
}
- c.Check(fs.memorySize(), check.Equals, int64(nDirs*67<<20))
+ c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67<<20))
c.Check(flushed, check.Equals, int64(0))
waitForFlush := func(expectUnflushed, expectFlushed int64) {
- for deadline := time.Now().Add(5 * time.Second); fs.memorySize() > expectUnflushed && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+ for deadline := time.Now().Add(5 * time.Second); fs.MemorySize() > expectUnflushed && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
}
- c.Check(fs.memorySize(), check.Equals, expectUnflushed)
+ c.Check(fs.MemorySize(), check.Equals, expectUnflushed)
c.Check(flushed, check.Equals, expectFlushed)
}
func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
+func (dn *deferrednode) MemorySize() int64 { return dn.currentinode().MemorySize() }
scopes: {type: 'array', required: false}
}
end
+
def create_system_auth
@object = ApiClientAuthorization.
new(user_id: system_user.id,
end
if Rails.configuration.Login.TokenLifetime > 0
if token_expiration == nil
- token_expiration = Time.now + Rails.configuration.Login.TokenLifetime
+ token_expiration = db_current_time + Rails.configuration.Login.TokenLifetime
else
- token_expiration = [token_expiration, Time.now + Rails.configuration.Login.TokenLifetime].min
+ token_expiration = [token_expiration, db_current_time + Rails.configuration.Login.TokenLifetime].min
end
end
include KindAndEtag
include CommonApiTemplate
extend CurrentApiClient
+ extend DbCurrentTime
belongs_to :api_client
belongs_to :user
after_initialize :assign_random_api_token
serialize :scopes, Array
+ before_validation :clamp_token_expiration
+
api_accessible :user, extend: :common do |t|
t.add :owner_uuid
t.add :user_id
auth.update_attributes!(user: user,
api_token: stored_secret,
api_client_id: 0,
- expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
+ expires_at: db_current_time + Rails.configuration.Login.RemoteTokenRefresh)
Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
auth.api_token = secret
return auth
protected
+ def clamp_token_expiration
+ if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
+ max_token_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime
+ if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ self.expires_at = max_token_expiration
+ end
+ end
+ end
+
def permission_to_create
current_user.andand.is_admin or (current_user.andand.id == self.user_id)
end
self.runtime_auth_scopes = ["all"]
end
- # generate a new token
+ # Generate a new token. This runs with admin credentials as it's done by a
+ # dispatcher user, so expires_at isn't enforced by API.MaxTokenLifetime.
self.auth = ApiClientAuthorization.
create!(user_id: User.find_by_uuid(self.runtime_user_uuid).id,
api_client_id: 0,
arvcfg.declare_config "API.MaxRequestSize", Integer, :max_request_size
arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
+arvcfg.declare_config "API.MaxTokenLifetime", ActiveSupport::Duration
arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
authorize_with :inactive
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
+ assert_response :redirect
assert_not_nil assigns(:api_client)
assert_nil assigns(:api_client_auth).expires_at
end
authorize_with :inactive
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
+ assert_response :redirect
assert_not_nil assigns(:api_client)
api_client_auth = assigns(:api_client_auth)
assert_in_delta(api_client_auth.expires_at,
require 'test_helper'
class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
+ extend DbCurrentTime
fixtures :all
test "create system auth" do
assert_response 403
end
+ [nil, db_current_time + 2.hours].each do |desired_expiration|
+ test "expires_at gets clamped on non-admins when API.MaxTokenLifetime is set and desired expires_at #{desired_expiration.nil? ? 'is not set' : 'exceeds the limit'}" do
+ Rails.configuration.API.MaxTokenLifetime = 1.hour
+
+ # Test token creation
+ start_t = db_current_time
+ post "/arvados/v1/api_client_authorizations",
+ params: {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:active).uuid,
+ :expires_at => desired_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ end_t = db_current_time
+ assert_response 200
+ expiration_t = json_response['expires_at'].to_time
+ assert_operator expiration_t.to_f, :>, (start_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ if !desired_expiration.nil?
+ assert_operator expiration_t.to_f, :<, desired_expiration.to_f
+ else
+ assert_operator expiration_t.to_f, :<, (end_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ end
+
+ # Test token update
+ previous_expiration = expiration_t
+ token_uuid = json_response["uuid"]
+ start_t = db_current_time
+ put "/arvados/v1/api_client_authorizations/#{token_uuid}",
+ params: {
+ :api_client_authorization => {
+ :expires_at => desired_expiration
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ end_t = db_current_time
+ assert_response 200
+ expiration_t = json_response['expires_at'].to_time
+ assert_operator previous_expiration.to_f, :<, expiration_t.to_f
+ assert_operator expiration_t.to_f, :>, (start_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ if !desired_expiration.nil?
+ assert_operator expiration_t.to_f, :<, desired_expiration.to_f
+ else
+ assert_operator expiration_t.to_f, :<, (end_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ end
+ end
+
+ test "expires_at can be set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
+ Rails.configuration.API.MaxTokenLifetime = 1.hour
+
+ # Test token creation
+ post "/arvados/v1/api_client_authorizations",
+ params: {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:admin).uuid,
+ :expires_at => desired_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response 200
+ if desired_expiration.nil?
+ assert json_response['expires_at'].nil?
+ else
+ assert_equal json_response['expires_at'].to_time.to_i, desired_expiration.to_i
+ end
+
+ # Test token update (reverse the above behavior)
+ previous_expiration = json_response['expires_at']
+ token_uuid = json_response['uuid']
+ if previous_expiration.nil?
+ desired_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
+ else
+ desired_updated_expiration = nil
+ end
+ put "/arvados/v1/api_client_authorizations/#{token_uuid}",
+ params: {
+ :api_client_authorization => {
+ :expires_at => desired_updated_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response 200
+ if desired_updated_expiration.nil?
+ assert json_response['expires_at'].nil?
+ else
+ assert_equal json_response['expires_at'].to_time.to_i, desired_updated_expiration.to_i
+ end
+ end
+ end
end
get("/arvados/v1/api_client_authorizations/current",
headers: authheaders)
assert_response 200
- #assert_not_empty json_response['uuid']
system_auth_uuid = json_response['uuid']
post("/arvados/v1/containers/#{containers(:queued).uuid}/lock",
test 'existing user login' do
mock_auth_with(identity_url: "https://active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'user redirect_to_user_uuid' do
mock_auth_with(identity_url: "https://redirects-to-active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'user double redirect_to_user_uuid' do
mock_auth_with(identity_url: "https://double-redirects-to-active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'create new user during omniauth callback' do
check_no_change_from_cancelled c
end
+ test "Container locked with non-expiring token" do
+ Rails.configuration.API.TokenMaxLifetime = 1.hour
+ set_user_from_auth :active
+ c, _ = minimal_new
+ set_user_from_auth :dispatch1
+ assert c.lock, show_errors(c)
+ refute c.auth.nil?
+ assert c.auth.expires_at.nil?
+ assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
+ end
+
test "Container locked cancel with log" do
set_user_from_auth :active
c, _ = minimal_new
import (
"sync"
+ "sync/atomic"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "github.com/hashicorp/golang-lru"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ lru "github.com/hashicorp/golang-lru"
"github.com/prometheus/client_golang/prometheus"
)
const metricsUpdateInterval = time.Second / 10
type cache struct {
- config *arvados.WebDAVCacheConfig
+ cluster *arvados.Cluster
+ config *arvados.WebDAVCacheConfig // TODO: use cluster.Collections.WebDAV instead
registry *prometheus.Registry
metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
+ sessions *lru.TwoQueueCache
setupOnce sync.Once
}
requests prometheus.Counter
collectionBytes prometheus.Gauge
collectionEntries prometheus.Gauge
+ sessionEntries prometheus.Gauge
collectionHits prometheus.Counter
pdhHits prometheus.Counter
permissionHits prometheus.Counter
+ sessionHits prometheus.Counter
+ sessionMisses prometheus.Counter
apiCalls prometheus.Counter
}
reg.MustRegister(m.apiCalls)
m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "arvados",
- Subsystem: "keepweb_collectioncache",
- Name: "cached_manifest_bytes",
- Help: "Total size of all manifests in cache.",
+ Subsystem: "keepweb_sessions",
+ Name: "cached_collection_bytes",
+ Help: "Total size of all cached manifests and sessions.",
})
reg.MustRegister(m.collectionBytes)
m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
Help: "Number of manifests in cache.",
})
reg.MustRegister(m.collectionEntries)
+ m.sessionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "active",
+ Help: "Number of active token sessions.",
+ })
+ reg.MustRegister(m.sessionEntries)
+ m.sessionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "hits",
+ Help: "Number of token session cache hits.",
+ })
+ reg.MustRegister(m.sessionHits)
+ m.sessionMisses = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "misses",
+ Help: "Number of token session cache misses.",
+ })
+ reg.MustRegister(m.sessionMisses)
}
type cachedPDH struct {
expire time.Time
}
+type cachedSession struct {
+ expire time.Time
+ fs atomic.Value
+}
+
func (c *cache) setup() {
var err error
c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries)
if err != nil {
panic(err)
}
+ c.sessions, err = lru.New2Q(c.config.MaxSessions)
+ if err != nil {
+ panic(err)
+ }
reg := c.registry
if reg == nil {
func (c *cache) updateGauges() {
c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
c.metrics.collectionEntries.Set(float64(c.collections.Len()))
+ c.metrics.sessionEntries.Set(float64(c.sessions.Len()))
}
var selectPDH = map[string]interface{}{
return err
}
+// ResetSession unloads any potentially stale state. Should be called
+// after write operations, so subsequent reads don't return stale
+// data.
+func (c *cache) ResetSession(token string) {
+ c.setupOnce.Do(c.setup)
+ c.sessions.Remove(token)
+}
+
+// Get a long-lived CustomFileSystem suitable for doing a read operation
+// with the given token.
+func (c *cache) GetSession(token string) (arvados.CustomFileSystem, error) {
+ c.setupOnce.Do(c.setup)
+ now := time.Now()
+ ent, _ := c.sessions.Get(token)
+ sess, _ := ent.(*cachedSession)
+ expired := false
+ if sess == nil {
+ c.metrics.sessionMisses.Inc()
+ sess = &cachedSession{
+ expire: now.Add(c.config.TTL.Duration()),
+ }
+ c.sessions.Add(token, sess)
+ } else if sess.expire.Before(now) {
+ c.metrics.sessionMisses.Inc()
+ expired = true
+ } else {
+ c.metrics.sessionHits.Inc()
+ }
+ go c.pruneSessions()
+ fs, _ := sess.fs.Load().(arvados.CustomFileSystem)
+ if fs != nil && !expired {
+ return fs, nil
+ }
+ ac, err := arvados.NewClientFromConfig(c.cluster)
+ if err != nil {
+ return nil, err
+ }
+ ac.AuthToken = token
+ arv, err := arvadosclient.New(ac)
+ if err != nil {
+ return nil, err
+ }
+ kc := keepclient.New(arv)
+ fs = ac.SiteFileSystem(kc)
+ fs.ForwardSlashNameSubstitution(c.cluster.Collections.ForwardSlashNameSubstitution)
+ sess.fs.Store(fs)
+ return fs, nil
+}
+
+// Remove all expired session cache entries, then remove more entries
+// until approximate remaining size <= maxsize/2
+func (c *cache) pruneSessions() {
+ now := time.Now()
+ var size int64
+ keys := c.sessions.Keys()
+ for _, token := range keys {
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ s := ent.(*cachedSession)
+ if s.expire.Before(now) {
+ c.sessions.Remove(token)
+ continue
+ }
+ if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
+ size += fs.MemorySize()
+ }
+ }
+ // Remove tokens until reaching size limit, starting with the
+ // least frequently used entries (which Keys() returns last).
+ for i := len(keys) - 1; i >= 0; i-- {
+ token := keys[i]
+ if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
+ break
+ }
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ s := ent.(*cachedSession)
+ fs, _ := s.fs.Load().(arvados.CustomFileSystem)
+ if fs == nil {
+ continue
+ }
+ c.sessions.Remove(token)
+ size -= fs.MemorySize()
+ }
+}
+
func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
c.setupOnce.Do(c.setup)
c.metrics.requests.Inc()
}
}
for i, k := range keys {
- if size <= c.config.MaxCollectionBytes {
+ if size <= c.config.MaxCollectionBytes/2 {
break
}
if expired[i] {
}
}
-// collectionBytes returns the approximate memory size of the
-// collection cache.
+// collectionBytes returns the approximate combined memory size of the
+// collection cache and session filesystem cache.
func (c *cache) collectionBytes() uint64 {
var size uint64
for _, k := range c.collections.Keys() {
}
size += uint64(len(v.(*cachedCollection).collection.ManifestText))
}
+ for _, token := range c.sessions.Keys() {
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ if fs, ok := ent.(*cachedSession).fs.Load().(arvados.CustomFileSystem); ok {
+ size += uint64(fs.MemorySize())
+ }
+ }
return size
}
func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosClient, kc *keepclient.KeepClient, client *arvados.Client, release func(), err error) {
arv = h.clientPool.Get()
if arv == nil {
- return nil, nil, nil, nil, err
+ err = h.clientPool.Err()
+ return
}
release = func() { h.clientPool.Put(arv) }
arv.ApiToken = token
http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
return
}
- _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), tokens[0])
+ fs, err := h.Config.Cache.GetSession(tokens[0])
if err != nil {
- http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- defer release()
-
- fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
f, err := fs.Open(r.URL.Path)
if os.IsNotExist(err) {
}
cfg.cluster = cls
cfg.Cache.config = &cfg.cluster.Collections.WebDAVCache
+ cfg.Cache.cluster = cls
return &cfg
}
normalizedURL := *r.URL
normalizedURL.RawPath = ""
normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+ ctxlog.FromContext(r.Context()).Infof("escapedPath %s", normalizedURL.EscapedPath())
canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
var InvalidRequest = "InvalidRequest"
var SignatureDoesNotMatch = "SignatureDoesNotMatch"
+var reRawQueryIndicatesAPI = regexp.MustCompile(`^[a-z]+(&|$)`)
+
// serveS3 handles r and returns true if r is a request from an S3
// client, otherwise it returns false.
func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
return false
}
- _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
- if err != nil {
- s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError)
- return true
+ var err error
+ var fs arvados.CustomFileSystem
+ if r.Method == http.MethodGet || r.Method == http.MethodHead {
+ // Use a single session (cached FileSystem) across
+ // multiple read requests.
+ fs, err = h.Config.Cache.GetSession(token)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
+ return true
+ }
+ } else {
+ // Create a FileSystem for this request, to avoid
+ // exposing incomplete write operations to concurrent
+ // requests.
+ _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
+ return true
+ }
+ defer release()
+ fs = client.SiteFileSystem(kc)
+ fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
}
- defer release()
-
- fs := client.SiteFileSystem(kc)
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
var objectNameGiven bool
var bucketName string
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
+ } else if _, ok = r.URL.Query()["location"]; ok {
+ // GetBucketLocation
+ w.Header().Set("Content-Type", "application/xml")
+ io.WriteString(w, xml.Header)
+ fmt.Fprintln(w, `<LocationConstraint><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
+ h.Config.cluster.ClusterID+
+ `</LocationConstraint></LocationConstraint>`)
+ } else if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // GetBucketWebsite ("GET /bucketid/?website"), GetBucketTagging, etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
} else {
// ListObjects
h.s3list(bucketName, w, r, fs)
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // GetObjectRetention ("GET /bucketid/objectid?retention&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
http.FileServer(fs).ServeHTTP(w, &r)
return true
case r.Method == http.MethodPut:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // PutObjectAcl ("PUT /bucketid/objectid?acl&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
if !objectNameGiven {
s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
+ // Ensure a subsequent read operation will see the changes.
+ h.Config.Cache.ResetSession(token)
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // DeleteObjectTagging ("DELETE /bucketid/objectid?tagging&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
if !objectNameGiven || r.URL.Path == "/" {
s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
+ // Ensure a subsequent read operation will see the changes.
+ h.Config.Cache.ResetSession(token)
w.WriteHeader(http.StatusNoContent)
return true
default:
s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed)
-
return true
}
}
auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
region := aws.Region{
- Name: s.testServer.Addr,
+ Name: "zzzzz",
S3Endpoint: "http://" + s.testServer.Addr,
}
client := s3.New(*auth, region)
}
func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
- scope := "20200202/region/service/aws4_request"
+ scope := "20200202/zzzzz/service/aws4_request"
signedHeaders := "date"
req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
{"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
} {
date := time.Now().UTC().Format("20060102T150405Z")
- scope := "20200202/fakeregion/S3/aws4_request"
+ scope := "20200202/zzzzz/S3/aws4_request"
canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
c.Logf("canonicalRequest %q", canonicalRequest)
expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
}
}
+func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
+ req, err := http.NewRequest("GET", bucket.URL("/"), nil)
+ c.Check(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = "location"
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
+ }
+}
+
func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
}
}
+func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ method string
+ path string
+ rawquery string
+ }{
+ {"GET", "/", "acl&versionId=1234"}, // GetBucketAcl
+ {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
+ {"PUT", "/", "acl"}, // PutBucketAcl
+ {"PUT", "/foo", "acl"}, // PutObjectAcl
+ {"DELETE", "/", "tagging"}, // DeleteBucketTagging
+ {"DELETE", "/foo", "tagging"}, // DeleteObjectTagging
+ } {
+ for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
+ c.Logf("trial %v bucket %v", trial, bucket)
+ req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
+ c.Check(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = trial.rawquery
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
+ }
+ }
+}
+
// If there are no CommonPrefixes entries, the CommonPrefixes XML tag
// should not appear at all.
func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
// FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
- c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
+ c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
if err != nil {
return nil, fmt.Errorf("error initializing volume %s: %s", uuid, err)
}
- logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+ logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly || va.ReadOnly)
sc := cfgvol.StorageClasses
if len(sc) == 0 {
require 'etc'
require 'fileutils'
require 'yaml'
+require 'optparse'
req_envs = %w(ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_VIRTUAL_MACHINE_UUID)
req_envs.each do |k|
end
end
-exclusive_mode = ARGV.index("--exclusive")
+options = {}
+OptionParser.new do |parser|
+ parser.on('--exclusive', 'Manage SSH keys file exclusively.')
+ parser.on('--rotate-tokens', 'Always create new user tokens. Usually needed with --token-lifetime.')
+ parser.on('--skip-missing-users', "Don't try to create any local accounts.")
+ parser.on('--token-lifetime SECONDS', 'Create user tokens that expire after SECONDS.', Integer)
+end.parse!(into: options)
+
exclusive_banner = "#######################################################################################
# THIS FILE IS MANAGED BY #{$0} -- CHANGES WILL BE OVERWRITTEN #
#######################################################################################\n\n"
start_banner = "### BEGIN Arvados-managed keys -- changes between markers will be overwritten\n"
end_banner = "### END Arvados-managed keys -- changes between markers will be overwritten\n"
-# Don't try to create any local accounts
-skip_missing_users = ARGV.index("--skip-missing-users")
-
keys = ''
begin
begin
pwnam[l[:username]] = Etc.getpwnam(l[:username])
rescue
- if skip_missing_users
+ if options[:"skip-missing-users"]
STDERR.puts "Account #{l[:username]} not found. Skipping"
true
end
oldkeys = ""
end
- if exclusive_mode
+ if options[:exclusive]
newkeys = exclusive_banner + newkeys
elsif oldkeys.start_with?(exclusive_banner)
newkeys = start_banner + newkeys + end_banner
tokenfile = File.join(configarvados, "settings.conf")
begin
- if !File.exist?(tokenfile)
- user_token = logincluster_arv.api_client_authorization.create(api_client_authorization: {owner_uuid: l[:user_uuid], api_client_id: 0})
+ if !File.exist?(tokenfile) || options[:"rotate-tokens"]
+ aca_params = {owner_uuid: l[:user_uuid], api_client_id: 0}
+ if options[:"token-lifetime"] && options[:"token-lifetime"] > 0
+ aca_params.merge!(expires_at: (Time.now + options[:"token-lifetime"]))
+ end
+ user_token = logincluster_arv.api_client_authorization.create(api_client_authorization: aca_params)
f = File.new(tokenfile, 'w')
f.write("ARVADOS_API_HOST=#{ENV['ARVADOS_API_HOST']}\n")
f.write("ARVADOS_API_TOKEN=v2/#{user_token[:uuid]}/#{user_token[:api_token]}\n")
{
"variables": {
+ "arvados_cluster": "",
+ "associate_public_ip_address": "true",
"aws_access_key": "",
- "aws_secret_key": "",
"aws_profile": "",
- "build_environment": "aws",
- "arvados_cluster": "",
+ "aws_secret_key": "",
"aws_source_ami": "ami-04d70e069399af2e9",
+ "build_environment": "aws",
+ "public_key_file": "",
+ "reposuffix": "",
+ "resolver": "",
"ssh_user": "admin",
- "vpc_id": "",
"subnet_id": "",
- "public_key_file": "",
- "associate_public_ip_address": "true"
+ "vpc_id": ""
},
"builders": [{
"type": "amazon-ebs",
{
"variables": {
- "resource_group": null,
+ "account_file": "",
+ "arvados_cluster": "",
+ "build_environment": "azure-arm",
"client_id": "{{env `ARM_CLIENT_ID`}}",
"client_secret": "{{env `ARM_CLIENT_SECRET`}}",
- "subscription_id": "{{env `ARM_SUBSCRIPTION_ID`}}",
- "tenant_id": "{{env `ARM_TENANT_ID`}}",
- "build_environment": "azure-arm",
"cloud_environment_name": "Public",
- "location": "centralus",
- "ssh_user": "packer",
- "ssh_private_key_file": "{{env `PACKERPRIVKEY`}}",
"image_sku": "",
- "arvados_cluster": "",
+ "location": "centralus",
"project_id": "",
- "account_file": "",
- "resolver": "",
+ "public_key_file": "",
"reposuffix": "",
- "public_key_file": ""
+ "resolver": "",
+ "resource_group": null,
+ "ssh_private_key_file": "{{env `PACKERPRIVKEY`}}",
+ "ssh_user": "packer",
+ "subscription_id": "{{env `ARM_SUBSCRIPTION_ID`}}",
+ "tenant_id": "{{env `ARM_TENANT_ID`}}"
},
"builders": [
{
Azure SKU image to use
--ssh_user (default: packer)
The user packer will use to log into the image
- --resolver (default: 8.8.8.8)
+ --resolver (default: host's network provided)
The dns resolver for the machine
--reposuffix (default: unset)
Set this to "-dev" to track the unstable/dev Arvados repositories
SUDO=sudo
+wait_for_apt_locks() {
+ while $SUDO fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+ echo "APT: Waiting for apt/dpkg locks to be released..."
+ sleep 1
+ done
+}
+
# Run apt-get update
$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
# Install gnupg and dirmgr or gpg key checks will fail
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
gnupg \
dirmngr \
lsb-release
# For good measure, apt-get upgrade
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes upgrade
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes upgrade
# Make sure cloud-init is installed
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install cloud-init
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install cloud-init
if [[ ! -d /var/lib/cloud/scripts/per-boot ]]; then
mkdir -p /var/lib/cloud/scripts/per-boot
fi
# Add the arvados signing key
cat /tmp/1078ECD7.asc | $SUDO apt-key add -
# Add the debian keys
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get install --yes debian-keyring debian-archive-keyring
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get install --yes debian-keyring debian-archive-keyring
# Fix locale
$SUDO /bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
$SUDO /usr/sbin/locale-gen
# Install some packages we always need
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
openssh-server \
apt-utils \
git \
cryptsetup \
xfsprogs
-# See if python3-distutils is installable, and if so install it. This is a
-# temporary workaround for an Arvados packaging bug and should be removed once
-# Arvados 2.0.4 or 2.1.0 is released, whichever comes first.
-# See https://dev.arvados.org/issues/16611 for more information
-if apt-cache -qq show python3-distutils >/dev/null 2>&1; then
- $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install python3-distutils
-fi
-
# Install the Arvados packages we need
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
- python-arvados-fuse \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+ python3-arvados-fuse \
crunch-run \
arvados-docker-cleaner \
docker.io
# Remove unattended-upgrades if it is installed
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes remove unattended-upgrades --purge
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes remove unattended-upgrades --purge
# Configure arvados-docker-cleaner
$SUDO mkdir -p /etc/arvados/docker-cleaner
$SUDO sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"/g' /etc/default/grub
$SUDO update-grub
-# Set a higher ulimit for docker
-$SUDO sed -i "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 --dns ${RESOLVER}/g" /lib/systemd/system/docker.service
+# Set a higher ulimit and the resolver (if set) for docker
+if [ "x$RESOLVER" != "x" ]; then
+ SET_RESOLVER="--dns ${RESOLVER}"
+fi
+
+$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
+ /lib/systemd/system/docker.service \
+ > /etc/systemd/system/docker.service
+
$SUDO systemctl daemon-reload
# Make sure user_allow_other is set in fuse.conf
$SUDO chmod 600 /home/crunch/.ssh/authorized_keys
$SUDO chmod 700 /home/crunch/.ssh/
-# Make sure we resolve via the provided resolver IP. Prepending is good enough because
+# Make sure we resolve via the provided resolver IP if set. Prepending is good enough because
# unless 'rotate' is set, the nameservers are queried in order (cf. man resolv.conf)
-$SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
-
+if [ "x$RESOLVER" != "x" ]; then
+ $SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
+fi
# Set up the cloud-init script that will ensure encrypted disks
$SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
$SUDO chmod 755 /usr/local/bin/ensure-encrypted-partitions.sh
echo YES | cryptsetup luksFormat "$LVPATH" "$KEYPATH"
cryptsetup --key-file "$KEYPATH" luksOpen "$LVPATH" "$(basename "$CRYPTPATH")"
shred -u "$KEYPATH"
-mkfs.xfs "$CRYPTPATH"
+mkfs.xfs -f "$CRYPTPATH"
# First make sure docker is not using /tmp, then unmount everything under it.
if [ -d /etc/sv/docker.io ]
projects / arvados.git / commitdiff
