13111: Merge branch 'master' into 12308-go-fuse

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg@veritasgenetics.com>

diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index 983c221f3b3b74f7235b04057cb3bd1bee1f9dba..c981b2a9ef679509bb24f7ecd25d005fb9d50920 100755 (executable)
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -161,6 +161,7 @@ popd
 
 if test -z "$packages" ; then
     packages="arvados-api-server
+        arvados-client
         arvados-docker-cleaner
         arvados-git-httpd
         arvados-node-manager

diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 54f8b0aed17e0f49d49d3ea26a6f3d0e2238355a..c56b74088f070c3c252b7c9f84e92eb5e3fea78c 100755 (executable)
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -341,6 +341,8 @@ fi
 cd $WORKSPACE/packages/$TARGET
 export GOPATH=$(mktemp -d)
 go get github.com/kardianos/govendor
+package_go_binary cmd/arvados-client arvados-client \
+    "Arvados command line tool (beta)"
 package_go_binary sdk/go/crunchrunner crunchrunner \
     "Crunchrunner executes a command inside a container and uploads the output"
 package_go_binary services/arv-git-httpd arvados-git-httpd \

diff --git a/build/run-tests.sh b/build/run-tests.sh
index f6c6e8f553fd93b0355cf9ce0224fc75495cc6d7..a02f732cb374e05f8dccfb3dadc9867c42e9456b 100755 (executable)
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -69,7 +69,11 @@ apps/workbench_functionals (*)
 apps/workbench_integration (*)
 apps/workbench_benchmark
 apps/workbench_profile
+cmd/arvados-client
 doc
+lib/cli
+lib/cmd
+lib/crunchstat
 services/api
 services/arv-git-httpd
 services/crunchstat
@@ -472,13 +476,36 @@ setup_virtualenv() {
 export PERLINSTALLBASE
 export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
 
+
 export GOPATH
 mkdir -p "$GOPATH/src/git.curoverse.com"
-rmdir --parents "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
-ln -snfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
+rmdir -v --parents --ignore-fail-on-non-empty "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH"
+for d in \
+    "$GOPATH/src/git.curoverse.com/arvados.git/arvados.git" \
+    "$GOPATH/src/git.curoverse.com/arvados.git"; do
+    [[ -d "$d" ]] && rmdir "$d"
+    [[ -h "$d" ]] && rm "$d"
+done
+ln -vsnfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
     || fatal "symlink failed"
 go get -v github.com/kardianos/govendor \
     || fatal "govendor install failed"
+cd "$GOPATH/src/git.curoverse.com/arvados.git" \
+    || fatal
+# Remove cached source dirs in workdir. Otherwise, they won't qualify
+# as +missing or +external below, and we won't be able to detect that
+# they're missing from vendor/vendor.json.
+rm -r vendor/*/
+go get -v -d ...
+"$GOPATH/bin/govendor" sync \
+    || fatal "govendor sync failed"
+[[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
+    || fatal "vendor/vendor.json has unused or missing dependencies -- try:
+* govendor remove +unused
+* govendor add +missing +external
+"
+cd "$WORKSPACE"
+
 
 setup_virtualenv "$VENVDIR" --python python2.7
 . "$VENVDIR/bin/activate"
@@ -824,11 +851,12 @@ install_apiserver() {
 }
 do_install services/api apiserver
 
-cd "$GOPATH/src/git.curoverse.com/arvados.git" && \
-    "$GOPATH/bin/govendor" sync -v || \
-        fatal "govendor sync failed"
 declare -a gostuff
 gostuff=(
+    cmd/arvados-client
+    lib/cli
+    lib/cmd
+    lib/crunchstat
     sdk/go/arvados
     sdk/go/arvadosclient
     sdk/go/blockdigest
@@ -839,7 +867,6 @@ gostuff=(
     sdk/go/asyncbuf
     sdk/go/crunchrunner
     sdk/go/stats
-    lib/crunchstat
     services/arv-git-httpd
     services/crunchstat
     services/health

diff --git a/cmd/arvados-client/.gitignore b/cmd/arvados-client/.gitignore
new file mode 100644 (file)
index 0000000..21dd863
--- /dev/null
+++ b/cmd/arvados-client/.gitignore
@@ -0,0 +1 @@
+arvados-*

diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
new file mode 100644 (file)
index 0000000..b616b54
--- /dev/null
+++ b/cmd/arvados-client/cmd.go
@@ -0,0 +1,79 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+       "fmt"
+       "io"
+       "os"
+       "regexp"
+       "runtime"
+
+       "git.curoverse.com/arvados.git/lib/cli"
+       "git.curoverse.com/arvados.git/lib/cmd"
+)
+
+var (
+       version                = "dev"
+       cmdVersion cmd.Handler = versionCmd{}
+       handler                = cmd.Multi(map[string]cmd.Handler{
+               "-e":        cmdVersion,
+               "version":   cmdVersion,
+               "-version":  cmdVersion,
+               "--version": cmdVersion,
+
+               "copy":     cli.Copy,
+               "create":   cli.Create,
+               "edit":     cli.Edit,
+               "get":      cli.Get,
+               "keep":     cli.Keep,
+               "pipeline": cli.Pipeline,
+               "run":      cli.Run,
+               "tag":      cli.Tag,
+               "ws":       cli.Ws,
+
+               "api_client_authorization": cli.APICall,
+               "api_client":               cli.APICall,
+               "authorized_key":           cli.APICall,
+               "collection":               cli.APICall,
+               "container":                cli.APICall,
+               "container_request":        cli.APICall,
+               "group":                    cli.APICall,
+               "human":                    cli.APICall,
+               "job":                      cli.APICall,
+               "job_task":                 cli.APICall,
+               "keep_disk":                cli.APICall,
+               "keep_service":             cli.APICall,
+               "link":                     cli.APICall,
+               "log":                      cli.APICall,
+               "node":                     cli.APICall,
+               "pipeline_instance":        cli.APICall,
+               "pipeline_template":        cli.APICall,
+               "repository":               cli.APICall,
+               "specimen":                 cli.APICall,
+               "trait":                    cli.APICall,
+               "user_agreement":           cli.APICall,
+               "user":                     cli.APICall,
+               "virtual_machine":          cli.APICall,
+               "workflow":                 cli.APICall,
+       })
+)
+
+type versionCmd struct{}
+
+func (versionCmd) RunCommand(prog string, args []string, _ io.Reader, stdout, _ io.Writer) int {
+       prog = regexp.MustCompile(` -*version$`).ReplaceAllLiteralString(prog, "")
+       fmt.Fprintf(stdout, "%s %s (%s)\n", prog, version, runtime.Version())
+       return 0
+}
+
+func fixLegacyArgs(args []string) []string {
+       flags, _ := cli.LegacyFlagSet()
+       return cmd.SubcommandToFront(args, flags)
+}
+
+func main() {
+       os.Exit(handler.RunCommand(os.Args[0], fixLegacyArgs(os.Args[1:]), os.Stdin, os.Stdout, os.Stderr))
+}

diff --git a/cmd/arvados-client/cmd_test.go b/cmd/arvados-client/cmd_test.go
new file mode 100644 (file)
index 0000000..cbbc7b1
--- /dev/null
+++ b/cmd/arvados-client/cmd_test.go
@@ -0,0 +1,41 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+       "bytes"
+       "io/ioutil"
+       "testing"
+
+       check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+       check.TestingT(t)
+}
+
+var _ = check.Suite(&ClientSuite{})
+
+type ClientSuite struct{}
+
+func (s *ClientSuite) TestBadCommand(c *check.C) {
+       exited := handler.RunCommand("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+       c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestBadSubcommandArgs(c *check.C) {
+       exited := handler.RunCommand("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
+       c.Check(exited, check.Equals, 2)
+}
+
+func (s *ClientSuite) TestVersion(c *check.C) {
+       stdout := bytes.NewBuffer(nil)
+       stderr := bytes.NewBuffer(nil)
+       exited := handler.RunCommand("arvados-client", []string{"version"}, bytes.NewReader(nil), stdout, stderr)
+       c.Check(exited, check.Equals, 0)
+       c.Check(stdout.String(), check.Matches, `arvados-client dev \(go[0-9\.]+\)\n`)
+       c.Check(stderr.String(), check.Equals, "")
+}

diff --git a/doc/_config.yml b/doc/_config.yml
index 7b992d6c9453ec87f43dd80fe044c94da2ad256b..08b581c0d71de7c3fe7ab831191735ee4e7c05b1 100644 (file)
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -74,7 +74,10 @@ navbar:
       - user/topics/run-command.html.textile.liquid
       - user/reference/job-pipeline-ref.html.textile.liquid
       - user/examples/crunch-examples.html.textile.liquid
+    - Admin tools:
       - user/topics/arvados-sync-groups.html.textile.liquid
+      - admin/change-account-owner.html.textile.liquid
+      - admin/merge-remote-account.html.textile.liquid
     - Query the metadata database:
       - user/topics/tutorial-trait-search.html.textile.liquid
     - Arvados License:

diff --git a/doc/admin/change-account-owner.html.textile.liquid b/doc/admin/change-account-owner.html.textile.liquid
new file mode 100644 (file)
index 0000000..d48572b
--- /dev/null
+++ b/doc/admin/change-account-owner.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: userguide
+title: "Changing account ownership"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+It is sometimes necessary to reassign an existing Arvados user account to a new Google account.
+
+Examples:
+* A user’s email address has changed from <code>person@old.example.com</code> to <code>person@new.example.com</code>.
+* A user who used to authenticate via LDAP is switching to Google login.
+
+This can be done by an administrator using Arvados APIs.
+
+First, determine the user’s existing UUID, e.g., @aaaaa-tpzed-abcdefghijklmno@.
+
+Ensure the new email address is not already associated with a different Arvados account. If it is, disassociate it by clearing that account’s @identity_url@ and @email@ fields.
+
+Clear the @identity_url@ field of the existing user record.
+
+Create a Link object with the following attributes (where @tail_uuid@ is the new email address, and @head_uuid@ is the existing user UUID):
+
+<notextile>
+<pre><code>{
+  "link_class":"permission",
+  "name":"can_login",
+  "tail_uuid":"<span class="userinput">person@new.example.com</span>",
+  "head_uuid":"<span class="userinput">aaaaa-tpzed-abcdefghijklmno</span>",
+  "properties":{
+    "identity_url_prefix":"https://www.google.com/"
+  }
+}
+</code></pre>
+</notextile>
+
+Have the user log in using their <code>person@new.example.com</code> Google account. You can verify this by checking that the @identity_url@ field has been populated.

diff --git a/doc/admin/merge-remote-account.html.textile.liquid b/doc/admin/merge-remote-account.html.textile.liquid
new file mode 100644 (file)
index 0000000..005332b
--- /dev/null
+++ b/doc/admin/merge-remote-account.html.textile.liquid
@@ -0,0 +1,45 @@
+---
+layout: default
+navsection: userguide
+title: "Merging a remote account"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+When you use federation capabilities to connect two or more clusters that were already operating, some users might already have accounts on multiple clusters. Typically, they will want to choose a single account on one of the clusters and abandon the rest, transferring all data or permissions from their old “remote” accounts to a single “home” account.
+
+This effect can be achieved by changing the UUIDs of the user records on the remote clusters. This should be done before the user has ever used federation features to access cluster B with cluster A credentials. Otherwise, see "managing conflicting accounts" below.
+
+For example, a user might have:
+* an account A on cluster A with uuid @aaaaa-tpzed-abcdefghijklmno@, and
+* an account B on cluster B with uuid @bbbbb-tpzed-lmnopqrstuvwxyz@
+
+An administrator at cluster B can merge the two accounts by renaming account B to account A.
+
+<notextile>
+<pre><code>#!/usr/bin/env python
+import arvados
+arvados.api('v1').users().update_uuid(
+    uuid="<span class="userinput">bbbbb-tpzed-lmnopqrstuvwxyz</span>",
+    new_uuid="<span class="userinput">aaaaa-tpzed-abcdefghijklmno</span>").execute()
+</code></pre></notextile>
+
+This should be done when the user is idle, i.e., not logged in and not running any jobs or containers.
+
+h2. Managing conflicting accounts
+
+If the user has already used federation capabilities to access cluster B using account A before the above migration has been done, this will have already created a database entry for account A on cluster B, and the above program will error out. To fix this, the same "update_uuid API call":../api/methods/users.html#update_uuid can be used to move the conflicting account out of the way first.
+
+<notextile>
+<pre><code>#!/usr/bin/env python
+import arvados
+import random
+import string
+random_chars = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(15))
+arvados.api('v1').users().update_uuid(
+    uuid="<span class="userinput">aaaaa-tpzed-abcdefghijklmno</span>",
+    new_uuid="bbbbb-tpzed-"+random_chars).execute()
+</code></pre></notextile>

diff --git a/doc/api/methods/users.html.textile.liquid b/doc/api/methods/users.html.textile.liquid
index 3a28e1e85610f4120aa3a4d4082c80c25edd8b3b..098c2ca118c71c85a9ce56576594e2d9b90926fb 100644 (file)
--- a/doc/api/methods/users.html.textile.liquid
+++ b/doc/api/methods/users.html.textile.liquid
@@ -111,3 +111,16 @@ table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 {background:#ccffcc}.|uuid|string|The UUID of the User in question.|path||
 |user|object||query||
+
+h3(#update_uuid). update_uuid
+
+Change the UUID of an existing user, updating all database references accordingly.
+
+This method can only be used by an admin user. It should only be used when the affected user is idle. New references to the affected user that are established _while the update_uuid operation is in progress_ might not be migrated as expected.
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+{background:#ccffcc}.|uuid|string|The current UUID of the user in question.|path|@zzzzz-tpzed-12345abcde12345@|
+{background:#ccffcc}.|new_uuid|string|The desired new UUID. It is an error to use a UUID belonging to an existing user.|query|@zzzzz-tpzed-abcde12345abcde@|

diff --git a/doc/install/crunch2-slurm/install-slurm.html.textile.liquid b/doc/install/crunch2-slurm/install-slurm.html.textile.liquid
index 589da07fda1422d6e75047a88f3fd982df1eec52..c69d18b8e4bd2b0b8e3a19802982fdc284eb0e42 100644 (file)
--- a/doc/install/crunch2-slurm/install-slurm.html.textile.liquid
+++ b/doc/install/crunch2-slurm/install-slurm.html.textile.liquid
@@ -85,6 +85,8 @@ Whenever you change this file, you will need to update the copy _on every comput
 
 *@ControlMachine@* should be a DNS name that resolves to the SLURM controller (dispatch/API server). This must resolve correctly on all SLURM worker nodes as well as the controller itself. In general SLURM is very sensitive about all of the nodes being able to communicate with the controller _and one another_, all using the same DNS names.
 
+*@SelectType=select/linear@* is needed on cloud-based installations that update node sizes dynamically, but it can only schedule one container at a time on each node. On a static or homogeneous cluster, use @SelectType=select/cons_res@ with @SelectTypeParameters=CR_CPU_Memory@ instead to enable node sharing.
+
 *@NodeName=compute[0-255]@* establishes that the hostnames of the worker nodes will be compute0, compute1, etc. through compute255.
 * There are several ways to compress sequences of names, like @compute[0-9,80,100-110]@. See the "hostlist" discussion in the @slurm.conf(5)@ and @scontrol(1)@ man pages for more information.
 * It is not necessary for all of the nodes listed here to be alive in order for SLURM to work, although you should make sure the DNS entries exist. It is easiest to define lots of hostnames up front, assigning them to real nodes and updating your DNS records as the nodes appear. This minimizes the frequency of @slurm.conf@ updates and use of @scontrol reconfigure@.

diff --git a/lib/cli/external.go b/lib/cli/external.go
index ba85aae9eaff654759c9eae1b3951264000d5cf2..35933f99fc2afafdb970cc2127587f182364f094 100644 (file)
--- a/lib/cli/external.go
+++ b/lib/cli/external.go
@@ -92,8 +92,33 @@ func (ec externalCmd) RunCommand(prog string, args []string, stdin io.Reader, st
                }
                fmt.Fprintf(stderr, "%s failed: %s\n", ec.prog, err)
                return 1
+       case *exec.Error:
+               fmt.Fprintln(stderr, err)
+               if ec.prog == "arv" || ec.prog == "arv-run-pipeline-instance" {
+                       fmt.Fprint(stderr, rubyInstallHints)
+               } else if strings.HasPrefix(ec.prog, "arv-") {
+                       fmt.Fprint(stderr, pythonInstallHints)
+               }
+               return 1
        default:
                fmt.Fprintf(stderr, "error running %s: %s\n", ec.prog, err)
                return 1
        }
 }
+
+var (
+       rubyInstallHints = `
+Note: This subcommand uses the arvados-cli Ruby gem. If that is not
+installed, try "gem install arvados-cli", or see
+https://doc.arvados.org/install for more details.
+
+`
+       pythonInstallHints = `
+Note: This subcommand uses the "arvados" Python module. If that is
+not installed, try:
+* "pip install arvados" (either as root or in a virtualenv), or
+* "sudo apt-get install python-arvados-python-client", or
+* see https://doc.arvados.org/install for more details.
+
+`
+)

diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go
index d04006586f307ee4636ebabfbaaae8c084654480..2cc71e68a8e749e76d558b6c98e9e2c26a23f2db 100644 (file)
--- a/lib/cmd/cmd.go
+++ b/lib/cmd/cmd.go
@@ -56,6 +56,11 @@ func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
 }
 
 func (m Multi) Usage(stderr io.Writer) {
+       fmt.Fprintf(stderr, "\nAvailable commands:\n")
+       m.listSubcommands(stderr, "")
+}
+
+func (m Multi) listSubcommands(out io.Writer, prefix string) {
        var subcommands []string
        for sc := range m {
                if strings.HasPrefix(sc, "-") {
@@ -67,9 +72,13 @@ func (m Multi) Usage(stderr io.Writer) {
                subcommands = append(subcommands, sc)
        }
        sort.Strings(subcommands)
-       fmt.Fprintf(stderr, "\nAvailable commands:\n")
        for _, sc := range subcommands {
-               fmt.Fprintf(stderr, "    %s\n", sc)
+               switch cmd := m[sc].(type) {
+               case Multi:
+                       cmd.listSubcommands(out, prefix+sc+" ")
+               default:
+                       fmt.Fprintf(out, "    %s%s\n", prefix, sc)
+               }
        }
 }
 

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 316af0e52847d2caad516ef1b6d126577b948e92..79bfacd28a10daad1393bf090f2350e7c166244c 100644 (file)
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,8 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
 
 import ruamel.yaml as yaml
 
-from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
+from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
+                     trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
 from .pathmapper import ArvPathMapper, trim_listing
 from .arvtool import ArvadosCommandTool
 from .perf import Perf
@@ -88,6 +89,8 @@ class ArvadosWorkflow(Workflow):
                     raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
             document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
 
+            discover_secondary_files(self.tool["inputs"], joborder)
+
             with Perf(metrics, "subworkflow upload_deps"):
                 upload_dependencies(self.arvrunner,
                                     os.path.basename(joborder.get("id", "#")),
@@ -155,7 +158,7 @@ class ArvadosWorkflow(Workflow):
                 "inputs": self.tool["inputs"],
                 "outputs": self.tool["outputs"],
                 "stdout": "cwl.output.json",
-                "requirements": workflowobj["requirements"]+[
+                "requirements": self.requirements+[
                     {
                     "class": "InitialWorkDirRequirement",
                     "listing": [{
@@ -169,7 +172,7 @@ class ArvadosWorkflow(Workflow):
                             "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
                         }]
                 }],
-                "hints": workflowobj["hints"],
+                "hints": self.hints,
                 "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"]
             })
             kwargs["loader"] = self.doc_loader

diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index c55e976924f65bd362153a0921152e5a170ca47b..28de7f368a23ccbc52a0cae37fa55d358744d0e1 100644 (file)
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -191,12 +191,8 @@ def tag_git_version(packed):
             packed["http://schema.org/version"] = githash
 
 
-def upload_job_order(arvrunner, name, tool, job_order):
-    """Upload local files referenced in the input object and return updated input
-    object with 'location' updated to the proper keep references.
-    """
-
-    for t in tool.tool["inputs"]:
+def discover_secondary_files(inputs, job_order):
+    for t in inputs:
         def setSecondary(fileobj):
             if isinstance(fileobj, dict) and fileobj.get("class") == "File":
                 if "secondaryFiles" not in fileobj:
@@ -209,6 +205,13 @@ def upload_job_order(arvrunner, name, tool, job_order):
         if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
             setSecondary(job_order[shortname(t["id"])])
 
+def upload_job_order(arvrunner, name, tool, job_order):
+    """Upload local files referenced in the input object and return updated input
+    object with 'location' updated to the proper keep references.
+    """
+
+    discover_secondary_files(tool.tool["inputs"], job_order)
+
     jobmapper = upload_dependencies(arvrunner,
                                     name,
                                     tool.doc_loader,

diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index f6271b85d2f9acab6f6371e3def545b1473cd8d7..d99cf6c189dc16761f73e5c1699ceed68baf9e9b 100644 (file)
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -123,3 +123,14 @@
     out: out
   tool: wf/runin-wf.cwl
   doc: "RunInSingleContainer cwl.input.json needs to be consistent with pathmapper manipulations"
+
+- job: secondary/wf-job.yml
+  output: {}
+  tool: secondary/wf.cwl
+  doc: "RunInSingleContainer applies secondaryFile discovery & manipulation before generating cwl.input.yml"
+
+- job: null
+  output:
+    out: out
+  tool: wf/runin-with-ttl-wf.cwl
+  doc: "RunInSingleContainer respects outputTTL"

diff --git a/sdk/cwl/tests/secondary/dir/hg19.fa b/sdk/cwl/tests/secondary/dir/hg19.fa
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/secondary/dir/hg19.fa.amb b/sdk/cwl/tests/secondary/dir/hg19.fa.amb
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/secondary/dir/hg19.fa.ann b/sdk/cwl/tests/secondary/dir/hg19.fa.ann
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/secondary/dir/hg19.fa.fai b/sdk/cwl/tests/secondary/dir/hg19.fa.fai
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/sdk/cwl/tests/secondary/ls.cwl b/sdk/cwl/tests/secondary/ls.cwl
new file mode 100644 (file)
index 0000000..b37990a
--- /dev/null
+++ b/sdk/cwl/tests/secondary/ls.cwl
@@ -0,0 +1,11 @@
+cwlVersion: v1.0
+class: CommandLineTool
+inputs:
+  i:
+    type: File
+    inputBinding:
+      position: 1
+    secondaryFiles:
+      - .fai
+outputs: []
+arguments: [ls, $(inputs.i), $(inputs.i.path).fai]

diff --git a/sdk/cwl/tests/secondary/sub.cwl b/sdk/cwl/tests/secondary/sub.cwl
new file mode 100644 (file)
index 0000000..5d2c699
--- /dev/null
+++ b/sdk/cwl/tests/secondary/sub.cwl
@@ -0,0 +1,17 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+inputs:
+  i:
+    type: File
+    secondaryFiles:
+      - .fai
+outputs: []
+steps:
+  step1:
+    in:
+      i: i
+    out: []
+    run: ls.cwl

diff --git a/sdk/cwl/tests/secondary/wf-job.yml b/sdk/cwl/tests/secondary/wf-job.yml
new file mode 100644 (file)
index 0000000..8b9dd83
--- /dev/null
+++ b/sdk/cwl/tests/secondary/wf-job.yml
@@ -0,0 +1,3 @@
+i:
+  class: File
+  location: keep:f225e6259bdd63bc7240599648dde9f1+97/hg19.fa

diff --git a/sdk/cwl/tests/secondary/wf.cwl b/sdk/cwl/tests/secondary/wf.cwl
new file mode 100644 (file)
index 0000000..248aefd
--- /dev/null
+++ b/sdk/cwl/tests/secondary/wf.cwl
@@ -0,0 +1,23 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+inputs:
+  i:
+    type: File
+    # secondaryFiles:
+    #   - .fai
+    #   - .ann
+    #   - .amb
+outputs: []
+steps:
+  step1:
+    in:
+      i: i
+    out: []
+    run: sub.cwl
+    requirements:
+      arv:RunInSingleContainer: {}

diff --git a/sdk/cwl/tests/wf/runin-with-ttl-wf.cwl b/sdk/cwl/tests/wf/runin-with-ttl-wf.cwl
new file mode 100644 (file)
index 0000000..713e0c0
--- /dev/null
+++ b/sdk/cwl/tests/wf/runin-with-ttl-wf.cwl
@@ -0,0 +1,63 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+inputs:
+  fileblub:
+    type: File
+    default:
+      class: File
+      location: keep:d7514270f356df848477718d58308cc4+94/a
+      secondaryFiles:
+        - class: File
+          location: keep:d7514270f356df848477718d58308cc4+94/b
+outputs:
+  out:
+    type: string
+    outputSource: substep/out
+requirements:
+  SubworkflowFeatureRequirement: {}
+  ScatterFeatureRequirement: {}
+  InlineJavascriptRequirement: {}
+  StepInputExpressionRequirement: {}
+hints:
+  arv:IntermediateOutput:
+    outputTTL: 60
+steps:
+  substep:
+    in:
+      fileblub: fileblub
+    out: [out]
+    hints:
+      - class: arv:RunInSingleContainer
+    run:
+      class: Workflow
+      id: mysub
+      inputs:
+        fileblub: File
+      outputs:
+        out:
+          type: string
+          outputSource: cat1/out
+      steps:
+        cat1:
+          in:
+            fileblub: fileblub
+          out: [out]
+          run:
+            class: CommandLineTool
+            id: subtool
+            inputs:
+              fileblub:
+                type: File
+                inputBinding: {position: 1}
+            outputs:
+              out:
+                type: string
+                outputBinding:
+                  outputEval: "out"
+            baseCommand: cat

diff --git a/sdk/go/arvados/node.go b/sdk/go/arvados/node.go
new file mode 100644 (file)
index 0000000..cc844fe
--- /dev/null
+++ b/sdk/go/arvados/node.go
@@ -0,0 +1,44 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "time"
+
+// Node is an arvados#node resource.
+type Node struct {
+       UUID       string         `json:"uuid"`
+       Domain     string         `json:"domain"`
+       Hostname   string         `json:"hostname"`
+       IPAddress  string         `json:"ip_address"`
+       LastPingAt *time.Time     `json:"last_ping_at,omitempty"`
+       SlotNumber int            `json:"slot_number"`
+       Status     string         `json:"status"`
+       JobUUID    string         `json:"job_uuid,omitempty"`
+       Properties NodeProperties `json:"properties"`
+}
+
+type NodeProperties struct {
+       CloudNode      NodePropertiesCloudNode `json:"cloud_node"`
+       TotalCPUCores  int                     `json:"total_cpu_cores,omitempty"`
+       TotalScratchMB int64                   `json:"total_scratch_mb,omitempty"`
+       TotalRAMMB     int64                   `json:"total_ram_mb,omitempty"`
+}
+
+type NodePropertiesCloudNode struct {
+       Size  string  `json:"size,omitempty"`
+       Price float64 `json:"price"`
+}
+
+func (c Node) resourceName() string {
+       return "node"
+}
+
+// NodeList is an arvados#nodeList resource.
+type NodeList struct {
+       Items          []Node `json:"items"`
+       ItemsAvailable int    `json:"items_available"`
+       Offset         int    `json:"offset"`
+       Limit          int    `json:"limit"`
+}

diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index 5e1235210a2a33d6d6c6b27c9daa36cf4a11e1bd..dc7e62f3e340741bec37e1d72bbb99c7ccf797d4 100644 (file)
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -9,7 +9,7 @@ class Arvados::V1::UsersController < ApplicationController
     [:activate, :current, :system, :setup]
   skip_before_filter :render_404_if_no_object, only:
     [:activate, :current, :system, :setup]
-  before_filter :admin_required, only: [:setup, :unsetup]
+  before_filter :admin_required, only: [:setup, :unsetup, :update_uuid]
 
   def current
     if current_user
@@ -118,6 +118,13 @@ class Arvados::V1::UsersController < ApplicationController
     show
   end
 
+  # Change UUID to a new (unused) uuid and transfer all owned/linked
+  # objects accordingly.
+  def update_uuid
+    @object.update_uuid(new_uuid: params[:new_uuid])
+    show
+  end
+
   protected
 
   def self._setup_requires_parameters
@@ -140,6 +147,14 @@ class Arvados::V1::UsersController < ApplicationController
     }
   end
 
+  def self._update_uuid_requires_parameters
+    {
+      new_uuid: {
+        type: 'string', required: true,
+      },
+    }
+  end
+
   def apply_filters(model_class=nil)
     return super if @read_users.any?(&:is_admin)
     if params[:uuid] != current_user.andand.uuid

diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index f3ca75929f7da09695759a7c92643d47a04158da..9209411f1e30fbb36ab0d44769c2815550bac0a0 100644 (file)
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -254,6 +254,32 @@ class User < ArvadosModel
     end
   end
 
+  def update_uuid(new_uuid:)
+    if !current_user.andand.is_admin
+      raise PermissionDeniedError
+    end
+    if uuid == system_user_uuid || uuid == anonymous_user_uuid
+      raise "update_uuid cannot update system accounts"
+    end
+    if self.class != self.class.resource_class_for_uuid(new_uuid)
+      raise "invalid new_uuid #{new_uuid.inspect}"
+    end
+    transaction(requires_new: true) do
+      reload
+      old_uuid = self.uuid
+      self.uuid = new_uuid
+      save!(validate: false)
+      ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+        klass.columns.each do |col|
+          if col.name.end_with?('_uuid')
+            column = col.name.to_sym
+            klass.where(column => old_uuid).update_all(column => new_uuid)
+          end
+        end
+      end
+    end
+  end
+
   protected
 
   def ensure_ownership_path_leads_to_user

diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 5b6fe80975faf0d5a56be3e17150d600ce525bc6..fcd5c34212d16187207e64abc8372e43ecd09af8 100644 (file)
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -79,6 +79,7 @@ Server::Application.routes.draw do
         post 'activate', on: :member
         post 'setup', on: :collection
         post 'unsetup', on: :member
+        post 'update_uuid', on: :member
       end
       resources :virtual_machines do
         get 'logins', on: :member

diff --git a/services/api/test/fixtures/api_client_authorizations.yml b/services/api/test/fixtures/api_client_authorizations.yml
index 8b283db45bf285674a0af9b6fa550453afb06e0a..8d9fc53c04d2c6bc4ccaea2009d72a178c6c11c7 100644 (file)
--- a/services/api/test/fixtures/api_client_authorizations.yml
+++ b/services/api/test/fixtures/api_client_authorizations.yml
@@ -300,14 +300,14 @@ running_container_auth:
   uuid: zzzzz-gj3su-077z32aux8dg2s2
   api_client: untrusted
   user: active
-  api_token: 3kg6k6lzmp9kj6bpkcoxie963cmvjahbt2fod9zru30k1jqdmi
+  api_token: it2gl94mgu3rbn5s2d06vzh73ns1y6cthct0tvg82qdlsxvbwk
   expires_at: 2038-01-01 00:00:00
 
-not_running_container_auth:
-  uuid: zzzzz-gj3su-077z32aux8dg2s3
+running_to_be_deleted_container_auth:
+  uuid: zzzzz-gj3su-ty6lvu9d7u7c2sq
   api_client: untrusted
   user: active
-  api_token: 4kg6k6lzmp9kj6bpkcoxie963cmvjahbt2fod9zru30k1jqdmj
+  api_token: ge1pez7dkk7nqntwcsj922g2b7a2t27xz6nsx39r15kbcqmp55
   expires_at: 2038-01-01 00:00:00
 
 permission_perftest:

diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index 7e2de6d8ca6207bc45738d4513507cae874dae58..39957916d3079be18da8c205b1d58bb928b30b37 100644 (file)
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -145,7 +145,6 @@ requester_container:
   runtime_constraints:
     ram: 12000000000
     vcpus: 4
-  auth_uuid: zzzzz-gj3su-077z32aux8dg2s3
 
 failed_container:
   uuid: zzzzz-dz642-failedcontainr1
@@ -232,4 +231,4 @@ running_to_be_deleted:
   runtime_constraints:
     ram: 12000000000
     vcpus: 4
-  auth_uuid: zzzzz-gj3su-077z32aux8dg2s2
+  auth_uuid: zzzzz-gj3su-ty6lvu9d7u7c2sq

diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index b75479ff8d145f0b11712273df0970f3f3ff6dc6..a50648617fd59aea8fb1bdb39c7066b3b4e60974 100644 (file)
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -794,6 +794,27 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
                     "user's writable_by should include its owner_uuid")
   end
 
+  [
+    [:admin, true],
+    [:active, false],
+  ].each do |auth_user, expect_success|
+    test "update_uuid as #{auth_user}" do
+      authorize_with auth_user
+      orig_uuid = users(:active).uuid
+      post :update_uuid, {
+             id: orig_uuid,
+             new_uuid: 'zbbbb-tpzed-abcde12345abcde',
+           }
+      if expect_success
+        assert_response :success
+        assert_empty User.where(uuid: orig_uuid)
+      else
+        assert_response 403
+        assert_not_empty User.where(uuid: orig_uuid)
+      end
+    end
+  end
+
 
   NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
                          "last_name"].sort

diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 3e2b8c1d3a550e16cb5da3be907e75a4383d37d4..ed86befbace2133bd06213b809279478418cf5a2 100644 (file)
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -584,7 +584,7 @@ class ContainerTest < ActiveSupport::TestCase
     c.lock
     c.update_attributes! state: Container::Running
 
-    set_user_from_auth :not_running_container_auth
+    set_user_from_auth :running_to_be_deleted_container_auth
     assert_raises ArvadosModel::PermissionDeniedError do
       c.update_attributes! output: collections(:foo_file).portable_data_hash
     end

diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 3d8fb9da7522f29d508fade008afa4c63d2692f9..72beca6c78134dbe92bd9ce4b65d8b3e70c6d530 100644 (file)
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -721,4 +721,83 @@ class UserTest < ActiveSupport::TestCase
     end
   end
 
+  [
+    [:active, 'zzzzz-borkd-abcde12345abcde'],
+    [:active, 'zzzzz-j7d0g-abcde12345abcde'],
+    [:active, 'zzzzz-tpzed-borkd'],
+    [:system_user, 'zzzzz-tpzed-abcde12345abcde'],
+    [:anonymous, 'zzzzz-tpzed-abcde12345abcde'],
+  ].each do |fixture, new_uuid|
+    test "disallow update_uuid #{fixture} -> #{new_uuid}" do
+      u = users(fixture)
+      orig_uuid = u.uuid
+      act_as_system_user do
+        assert_raises do
+          u.update_uuid(new_uuid: new_uuid)
+        end
+      end
+      # "Successfully aborted orig->new" outcome looks the same as
+      # "successfully updated new->orig".
+      assert_update_success(old_uuid: new_uuid,
+                            new_uuid: orig_uuid,
+                            expect_owned_objects: fixture == :active)
+    end
+  end
+
+  [:active, :spectator, :admin].each do |target|
+    test "update_uuid on #{target} as non-admin user" do
+      act_as_user users(:active) do
+        assert_raises(ArvadosModel::PermissionDeniedError) do
+          users(target).update_uuid(new_uuid: 'zzzzz-tpzed-abcde12345abcde')
+        end
+      end
+    end
+  end
+
+  test "update_uuid to existing uuid" do
+    u = users(:active)
+    orig_uuid = u.uuid
+    new_uuid = users(:admin).uuid
+    act_as_system_user do
+      assert_raises do
+        u.update_uuid(new_uuid: new_uuid)
+      end
+    end
+    u.reload
+    assert_equal u.uuid, orig_uuid
+    assert_not_empty Collection.where(owner_uuid: orig_uuid)
+    assert_not_empty Group.where(owner_uuid: orig_uuid)
+  end
+
+  [
+    [:active, 'zbbbb-tpzed-abcde12345abcde'],
+    [:active, 'zzzzz-tpzed-abcde12345abcde'],
+    [:admin, 'zbbbb-tpzed-abcde12345abcde'],
+    [:admin, 'zzzzz-tpzed-abcde12345abcde'],
+  ].each do |fixture, new_uuid|
+    test "update_uuid #{fixture} to unused uuid #{new_uuid}" do
+      u = users(fixture)
+      orig_uuid = u.uuid
+      act_as_system_user do
+        u.update_uuid(new_uuid: new_uuid)
+      end
+      assert_update_success(old_uuid: orig_uuid,
+                            new_uuid: new_uuid,
+                            expect_owned_objects: fixture == :active)
+    end
+  end
+
+  def assert_update_success(old_uuid:, new_uuid:, expect_owned_objects: true)
+    [[User, :uuid],
+     [Link, :head_uuid],
+     [Link, :tail_uuid],
+     [Group, :owner_uuid],
+     [Collection, :owner_uuid],
+    ].each do |klass, attr|
+      assert_empty klass.where(attr => old_uuid)
+      if klass == User || expect_owned_objects
+        assert_not_empty klass.where(attr => new_uuid)
+      end
+    end
+  end
 end

diff --git a/services/arv-git-httpd/arvados-git-httpd.service b/services/arv-git-httpd/arvados-git-httpd.service
index 300b5c1d12287d9e0c771e89f01eb9b4e92c6f5f..6f8cca856b595498b7f6b8ee387e956e838666a2 100644 (file)
--- a/services/arv-git-httpd/arvados-git-httpd.service
+++ b/services/arv-git-httpd/arvados-git-httpd.service
@@ -7,9 +7,11 @@ Description=Arvados git server
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/git-httpd/git-httpd.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/arvados-git-httpd
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
index ac842433d56ea8f379d6e44c4c04fb9d97e1fd7d..508a0dfc1387bafcca41b6b8d851f0960d3fc9b9 100644 (file)
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
@@ -7,9 +7,11 @@ Description=Arvados Crunch Dispatcher for SLURM
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/crunch-dispatch-slurm/crunch-dispatch-slurm.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/crunch-dispatch-slurm
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index e5e0ea001669f21219c8f6d01717f7771b885f59..6a118a67435e5166b5e27f62fe02ebffe237fdb3 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -660,9 +660,12 @@ type infoCommand struct {
        cmd   []string
 }
 
-// LogNodeInfo gathers node information and store it on the log for debugging
-// purposes.
-func (runner *ContainerRunner) LogNodeInfo() (err error) {
+// LogHostInfo logs info about the current host, for debugging and
+// accounting purposes. Although it's logged as "node-info", this is
+// about the environment where crunch-run is actually running, which
+// might differ from what's described in the node record (see
+// LogNodeRecord).
+func (runner *ContainerRunner) LogHostInfo() (err error) {
        w := runner.NewLogWriter("node-info")
 
        commands := []infoCommand{
@@ -710,38 +713,71 @@ func (runner *ContainerRunner) LogNodeInfo() (err error) {
 }
 
 // LogContainerRecord gets and saves the raw JSON container record from the API server
-func (runner *ContainerRunner) LogContainerRecord() (err error) {
+func (runner *ContainerRunner) LogContainerRecord() error {
+       logged, err := runner.logAPIResponse("container", "containers", map[string]interface{}{"filters": [][]string{{"uuid", "=", runner.Container.UUID}}}, nil)
+       if !logged && err == nil {
+               err = fmt.Errorf("error: no container record found for %s", runner.Container.UUID)
+       }
+       return err
+}
+
+// LogNodeRecord logs arvados#node record corresponding to the current host.
+func (runner *ContainerRunner) LogNodeRecord() error {
+       hostname := os.Getenv("SLURMD_NODENAME")
+       if hostname == "" {
+               hostname, _ = os.Hostname()
+       }
+       _, err := runner.logAPIResponse("node", "nodes", map[string]interface{}{"filters": [][]string{{"hostname", "=", hostname}}}, func(resp interface{}) {
+               // The "info" field has admin-only info when obtained
+               // with a privileged token, and should not be logged.
+               node, ok := resp.(map[string]interface{})
+               if ok {
+                       delete(node, "info")
+               }
+       })
+       return err
+}
+
+func (runner *ContainerRunner) logAPIResponse(label, path string, params map[string]interface{}, munge func(interface{})) (logged bool, err error) {
        w := &ArvLogWriter{
                ArvClient:     runner.ArvClient,
                UUID:          runner.Container.UUID,
-               loggingStream: "container",
-               writeCloser:   runner.LogCollection.Open("container.json"),
+               loggingStream: label,
+               writeCloser:   runner.LogCollection.Open(label + ".json"),
        }
 
-       // Get Container record JSON from the API Server
-       reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
+       reader, err := runner.ArvClient.CallRaw("GET", path, "", "", arvadosclient.Dict(params))
        if err != nil {
-               return fmt.Errorf("While retrieving container record from the API server: %v", err)
+               return false, fmt.Errorf("error getting %s record: %v", label, err)
        }
        defer reader.Close()
 
        dec := json.NewDecoder(reader)
        dec.UseNumber()
-       var cr map[string]interface{}
-       if err = dec.Decode(&cr); err != nil {
-               return fmt.Errorf("While decoding the container record JSON response: %v", err)
+       var resp map[string]interface{}
+       if err = dec.Decode(&resp); err != nil {
+               return false, fmt.Errorf("error decoding %s list response: %v", label, err)
+       }
+       items, ok := resp["items"].([]interface{})
+       if !ok {
+               return false, fmt.Errorf("error decoding %s list response: no \"items\" key in API list response", label)
+       } else if len(items) < 1 {
+               return false, nil
+       }
+       if munge != nil {
+               munge(items[0])
        }
        // Re-encode it using indentation to improve readability
        enc := json.NewEncoder(w)
        enc.SetIndent("", "    ")
-       if err = enc.Encode(cr); err != nil {
-               return fmt.Errorf("While logging the JSON container record: %v", err)
+       if err = enc.Encode(items[0]); err != nil {
+               return false, fmt.Errorf("error logging %s record: %v", label, err)
        }
        err = w.Close()
        if err != nil {
-               return fmt.Errorf("While closing container.json log: %v", err)
+               return false, fmt.Errorf("error closing %s.json in log collection: %v", label, err)
        }
-       return nil
+       return true, nil
 }
 
 // AttachStreams connects the docker container stdin, stdout and stderr logs
@@ -1552,13 +1588,14 @@ func (runner *ContainerRunner) Run() (err error) {
        if err != nil {
                return
        }
-
-       // Gather and record node information
-       err = runner.LogNodeInfo()
+       err = runner.LogHostInfo()
+       if err != nil {
+               return
+       }
+       err = runner.LogNodeRecord()
        if err != nil {
                return
        }
-       // Save container.json record on log collection
        err = runner.LogContainerRecord()
        if err != nil {
                return

diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index ab7417e542bee44fa2346e50a97732ec3b41304b..4979cf8a0c801ee5bc56e48933e97f736dc3d00d 100644 (file)
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -244,8 +244,23 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par
 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
        parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
        var j []byte
-       if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
-               j, err = json.Marshal(client.Container)
+       if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
+               j = []byte(`{
+                       "kind": "arvados#nodeList",
+                       "items": [{
+                               "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
+                               "hostname": "compute2",
+                               "properties": {"total_cpu_cores": 16}
+                       }]}`)
+       } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
+               if uuid == "" {
+                       j, err = json.Marshal(map[string]interface{}{
+                               "items": []interface{}{client.Container},
+                               "kind":  "arvados#nodeList",
+                       })
+               } else {
+                       j, err = json.Marshal(client.Container)
+               }
        } else {
                j = []byte(`{
                        "command": ["sleep", "1"],
@@ -768,6 +783,7 @@ func (s *TestSuite) TestCrunchstat(c *C) {
 }
 
 func (s *TestSuite) TestNodeInfoLog(c *C) {
+       os.Setenv("SLURMD_NODENAME", "compute2")
        api, _, _ := FullRunHelper(c, `{
                "command": ["sleep", "1"],
                "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
@@ -786,12 +802,19 @@ func (s *TestSuite) TestNodeInfoLog(c *C) {
        c.Check(api.CalledWith("container.exit_code", 0), NotNil)
        c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 
+       c.Assert(api.Logs["node"], NotNil)
+       json := api.Logs["node"].String()
+       c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
+       c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
+       c.Check(json, Not(Matches), `(?ms).*"info":.*`)
+
        c.Assert(api.Logs["node-info"], NotNil)
-       c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
-       c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
-       c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
-       c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
-       c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
+       json = api.Logs["node-info"].String()
+       c.Check(json, Matches, `(?ms).*Host Information.*`)
+       c.Check(json, Matches, `(?ms).*CPU Information.*`)
+       c.Check(json, Matches, `(?ms).*Memory Information.*`)
+       c.Check(json, Matches, `(?ms).*Disk Space.*`)
+       c.Check(json, Matches, `(?ms).*Disk INodes.*`)
 }
 
 func (s *TestSuite) TestContainerRecordLog(c *C) {

diff --git a/services/crunch-run/git_mount.go b/services/crunch-run/git_mount.go
index 92b83710b13e9966bfba521214c1bbc50f48fd31..2ecc8726f5e54d91518b3f20c08eb8f1dec41852 100644 (file)
--- a/services/crunch-run/git_mount.go
+++ b/services/crunch-run/git_mount.go
@@ -8,7 +8,7 @@ import (
        "regexp"
 
        "git.curoverse.com/arvados.git/sdk/go/arvados"
-       "gopkg.in/src-d/go-billy.v3/osfs"
+       "gopkg.in/src-d/go-billy.v4/osfs"
        git "gopkg.in/src-d/go-git.v4"
        git_config "gopkg.in/src-d/go-git.v4/config"
        git_plumbing "gopkg.in/src-d/go-git.v4/plumbing"
@@ -78,7 +78,10 @@ func (gm gitMount) extractTree(ac IArvadosClient, dir string, token string) erro
        }
        err = repo.Fetch(&git.FetchOptions{
                RemoteName: "origin",
-               Auth:       git_http.NewBasicAuth("none", token),
+               Auth: &git_http.BasicAuth{
+                       Username: "none",
+                       Password: token,
+               },
        })
        if err != nil {
                return fmt.Errorf("git fetch %q: %s", u.String(), err)

diff --git a/services/dockercleaner/arvados-docker-cleaner.service b/services/dockercleaner/arvados-docker-cleaner.service
index 6ab3fdfac91ae05e183643b010ca6d093a4b2f3c..b48b46bb9a4c48b52770996fba12821c60e5cb8e 100644 (file)
--- a/services/dockercleaner/arvados-docker-cleaner.service
+++ b/services/dockercleaner/arvados-docker-cleaner.service
@@ -7,9 +7,11 @@ Description=Arvados Docker Image Cleaner
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/docker-cleaner/docker-cleaner.json
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -23,5 +25,8 @@ RestartPreventExitStatus=2
 # Collection, and if so, invokes it with the "scl" wrapper.
 ExecStart=/bin/sh -c 'if [ -e /opt/rh/python33/root/bin/arvados-docker-cleaner ]; then exec scl enable python33 arvados-docker-cleaner; else exec arvados-docker-cleaner; fi'
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/health/arvados-health.service b/services/health/arvados-health.service
new file mode 100644 (file)
index 0000000..dac7c3a
--- /dev/null
+++ b/services/health/arvados-health.service
@@ -0,0 +1,27 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+[Unit]
+Description=Arvados healthcheck server
+Documentation=https://doc.arvados.org/
+After=network.target
+AssertPathExists=/etc/arvados/config.yml
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
+StartLimitInterval=0
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/arvados-health
+Restart=always
+RestartSec=1
+
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target

diff --git a/services/keep-balance/keep-balance.service b/services/keep-balance/keep-balance.service
index df08b5be7f2e3f75414d8b2fa8f46bb08bb4c844..325affe5875108a819b3baa07aa964bcd5ef1224 100644 (file)
--- a/services/keep-balance/keep-balance.service
+++ b/services/keep-balance/keep-balance.service
@@ -7,9 +7,11 @@ Description=Arvados Keep Balance
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keep-balance/keep-balance.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
 Restart=always
 RestartSec=10s
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/keep-web/keep-web.service b/services/keep-web/keep-web.service
index a8f8f1ffa1336329d7ff0f0c6ad76daeb65ddb0d..1931256209eb4f211d031114c8297fb0ee53d01f 100644 (file)
--- a/services/keep-web/keep-web.service
+++ b/services/keep-web/keep-web.service
@@ -7,9 +7,11 @@ Description=Arvados Keep web gateway
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keep-web/keep-web.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keep-web
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/keepproxy/keepproxy.service b/services/keepproxy/keepproxy.service
index d7ee97e96325ba278ec75eba6e9f80dedf41f5a3..96dec25ecf77ecc8c3936628829d8dc59aecabc4 100644 (file)
--- a/services/keepproxy/keepproxy.service
+++ b/services/keepproxy/keepproxy.service
@@ -7,9 +7,11 @@ Description=Arvados Keep Proxy
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keepproxy/keepproxy.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keepproxy
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index bb0e9bbf6874859211fefea70955472cc797afb3..de72e747f3fa10c1cc1c3d64839ec317c29d42e3 100644 (file)
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -355,7 +355,7 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
                _, _, err := kc.Ask(hash)
                errNotFound, _ := err.(keepclient.ErrNotFound)
                c.Check(errNotFound, NotNil)
-               c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
+               c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
                c.Log("Ask 1")
        }
 
@@ -371,7 +371,7 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
                blocklen, _, err := kc.Ask(hash)
                errNotFound, _ := err.(keepclient.ErrNotFound)
                c.Check(errNotFound, NotNil)
-               c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
+               c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
                c.Check(blocklen, Equals, int64(0))
                c.Log("Ask 2")
        }
@@ -380,7 +380,7 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
                _, blocklen, _, err := kc.Get(hash)
                errNotFound, _ := err.(keepclient.ErrNotFound)
                c.Check(errNotFound, NotNil)
-               c.Assert(strings.Contains(err.Error(), "HTTP 400"), Equals, true)
+               c.Assert(err, ErrorMatches, `.*HTTP 405.*`)
                c.Check(blocklen, Equals, int64(0))
                c.Log("Get")
        }
@@ -596,14 +596,14 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
        c.Check(err, NotNil)
        errNotFound, _ := err.(*keepclient.ErrNotFound)
        c.Check(errNotFound.Temporary(), Equals, true)
-       c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
+       c.Assert(err, ErrorMatches, ".*connection refused.*")
 
        // Get should result in temporary connection refused error
        _, _, _, err = kc.Get(hash)
        c.Check(err, NotNil)
        errNotFound, _ = err.(*keepclient.ErrNotFound)
        c.Check(errNotFound.Temporary(), Equals, true)
-       c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true)
+       c.Assert(err, ErrorMatches, ".*connection refused.*")
 }
 
 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
@@ -644,5 +644,5 @@ func (s *ServerRequiredSuite) TestPing(c *C) {
        resp := httptest.NewRecorder()
        rtr.ServeHTTP(resp, req)
        c.Check(resp.Code, Equals, 200)
-       c.Assert(strings.Contains(resp.Body.String(), `{"health":"OK"}`), Equals, true)
+       c.Assert(resp.Body.String(), Matches, `{"health":"OK"}\n?`)
 }

diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index 4d042a70dd376ea1e04bdff16283ab80669dfd0f..f012ea3902217c55502d7aebe12b072d004985ec 100644 (file)
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -1107,7 +1107,7 @@ func TestUntrashHandler(t *testing.T) {
        response = IssueRequest(datamanagerWrongMethodReq)
        ExpectStatusCode(t,
                "Only PUT method is supported for untrash",
-               http.StatusBadRequest,
+               http.StatusMethodNotAllowed,
                response)
 
        // datamanagerReq => StatusOK

diff --git a/services/keepstore/keepstore.service b/services/keepstore/keepstore.service
index 14572ae905a1d257b08e404f0dbcde7b3b88715d..8b448e72c3b41687f46dc2c6a6ad8a0c21202258 100644 (file)
--- a/services/keepstore/keepstore.service
+++ b/services/keepstore/keepstore.service
@@ -7,9 +7,11 @@ Description=Arvados Keep Storage Daemon
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/keepstore/keepstore.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/keepstore
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/services/ws/arvados-ws.service b/services/ws/arvados-ws.service
index 4385261181922e0edc260e680a9678d7071651dc..9e02d41ca8bb6212684a3fd33b228b7fbffe390f 100644 (file)
--- a/services/ws/arvados-ws.service
+++ b/services/ws/arvados-ws.service
@@ -7,9 +7,11 @@ Description=Arvados websocket server
 Documentation=https://doc.arvados.org/
 After=network.target
 AssertPathExists=/etc/arvados/ws/ws.yml
-# systemd<230
+
+# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section
 StartLimitInterval=0
-# systemd>=230
+
+# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section
 StartLimitIntervalSec=0
 
 [Service]
@@ -18,5 +20,8 @@ ExecStart=/usr/bin/arvados-ws
 Restart=always
 RestartSec=1
 
+# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
+StartLimitInterval=0
+
 [Install]
 WantedBy=multi-user.target

diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index f4a65da537ecb96f861c946ba8c92a2938dd1600..234d15aecd03d42537b5f93f02a4dcd8b3f0340b 100755 (executable)
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -220,8 +220,8 @@ run() {
                        "--volume=$PIPCACHE:/var/lib/pip:rw" \
                        "--volume=$NPMCACHE:/var/lib/npm:rw" \
                        "--volume=$GOSTUFF:/var/lib/gopath:rw" \
-                       arvados/arvbox-dev$TAG \
-                       /usr/local/bin/runsvinit -svdir=/etc/test-service
+                      "--env=SVDIR=/etc/test-service" \
+                       arvados/arvbox-dev$TAG
 
                 docker exec -ti \
                        $ARVBOX_CONTAINER \

diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index f29066ec9fcb71de007837e295de61bcefc85ffa..83d507b62b4931163b73dded52240705ebd2ae70 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -62,10 +62,6 @@ RUN mkdir -p /etc/apt/sources.list.d && \
 
 RUN rm -rf /var/lib/postgresql && mkdir -p /var/lib/postgresql
 
-RUN cd /root && \
-    GOPATH=$PWD go get github.com/curoverse/runsvinit && \
-    install bin/runsvinit /usr/local/bin
-
 ENV PJSVERSION=1.9.8
 # bitbucket is the origin, but downloads fail sometimes, so use our own mirror instead.
 #ENV PJSURL=https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-${PJSVERSION}-linux-x86_64.tar.bz2
@@ -96,8 +92,9 @@ ADD crunch-setup.sh gitolite.rc \
     application_yml_override.py api-setup.sh \
     /usr/local/lib/arvbox/
 
-RUN mkdir /etc/docker
-ADD daemon.json /etc/docker/
+ADD runit /etc/runit
 
 # Start the supervisor.
-CMD ["/usr/local/bin/runsvinit"]
+ENV SVDIR /etc/service
+STOPSIGNAL SIGINT
+CMD ["/sbin/runit"]

diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index c5e15233bcb9720a7a8f7d416a8bf274e42fa623..2cb8473657e7705bc0759bd054da0e0d57222754 100644 (file)
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -46,17 +46,16 @@ run_bundler() {
     else
         frozen=""
     fi
-    if ! test -x bundle ; then
+    if ! test -x /var/lib/gems/bin/bundler ; then
         bundlergem=$(ls -r $GEM_HOME/cache/bundler-*.gem 2>/dev/null | head -n1 || true)
         if test -n "$bundlergem" ; then
             flock /var/lib/gems/gems.lock gem install --local --no-document $bundlergem
         else
             flock /var/lib/gems/gems.lock gem install --no-document bundler
         fi
-        ln -sf /var/lib/gems/bin/bundle /usr/local/bin
     fi
-    if ! flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
-        flock /var/lib/gems/gems.lock bundle install --path $GEM_HOME --no-deployment $frozen "$@"
+    if ! flock /var/lib/gems/gems.lock bundler install --path $GEM_HOME --local --no-deployment $frozen "$@" ; then
+        flock /var/lib/gems/gems.lock bundler install --path $GEM_HOME --no-deployment $frozen "$@"
     fi
 }
 

diff --git a/tools/arvbox/lib/arvbox/docker/runit/1 b/tools/arvbox/lib/arvbox/docker/runit/1
new file mode 100755 (executable)
index 0000000..fea9cf6
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/runit/1
@@ -0,0 +1,7 @@
+#!/bin/sh
+# system one time tasks
+
+PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
+
+touch /run/runit.stopit
+chmod 0 /run/runit.stopit

diff --git a/tools/arvbox/lib/arvbox/docker/runit/2 b/tools/arvbox/lib/arvbox/docker/runit/2
new file mode 100755 (executable)
index 0000000..6b092ea
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/runit/2
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+PATH=/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin:/usr/X11R6/bin
+
+echo
+echo "Arvados-in-a-box starting"
+echo
+
+exec env - PATH=$PATH \
+runsvdir -P $SVDIR

diff --git a/tools/arvbox/lib/arvbox/docker/runit/3 b/tools/arvbox/lib/arvbox/docker/runit/3
new file mode 100755 (executable)
index 0000000..525b96b
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/runit/3
@@ -0,0 +1,14 @@
+#!/bin/sh
+exec 2>&1
+
+PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
+
+LAST=0
+test -x /run/runit.reboot && LAST=6
+
+echo 'Waiting for services to stop...'
+sv -w196 force-stop /service/*
+sv exit /service/*
+
+echo 'Shutdown...'
+/etc/init.d/rc $LAST

diff --git a/tools/arvbox/lib/arvbox/docker/runit/ctrlaltdel b/tools/arvbox/lib/arvbox/docker/runit/ctrlaltdel
new file mode 100755 (executable)
index 0000000..02bb2ea
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/runit/ctrlaltdel
@@ -0,0 +1,13 @@
+#!/bin/sh
+exec 2>&1
+
+PATH=/command:/sbin:/bin:/usr/sbin:/usr/bin
+
+LAST=0
+test -x /run/runit.reboot && LAST=6
+
+echo 'Waiting for services to stop...'
+sv -w196 force-stop $SVDIR/*
+sv exit $SVDIR/*
+
+echo 'Shutdown...'

diff --git a/tools/arvbox/lib/arvbox/docker/service/docker/run b/tools/arvbox/lib/arvbox/docker/service/docker/run
index cbc8486d6ee88f23afb516ed81e98e4bd89469f7..b6dbaf1abb2db174ccceca73258bd0727ca850e2 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/docker/run
+++ b/tools/arvbox/lib/arvbox/docker/service/docker/run
@@ -100,6 +100,12 @@ rm -rf /var/run/docker.pid
 read pid cmd state ppid pgrp session tty_nr tpgid rest < /proc/self/stat
 trap "kill -TERM -$pgrp; exit" EXIT TERM KILL SIGKILL SIGTERM SIGQUIT
 
-if ! docker daemon --storage-driver=overlay $DOCKER_DAEMON_ARGS ; then
-    docker daemon $DOCKER_DAEMON_ARGS
+mkdir /etc/docker
+# Prefer overlay2
+echo '{"storage-driver": "overlay2"}' > /etc/docker/daemon.json
+
+if ! dockerd ; then
+    # Oops overlay2 didn't work, let docker choose a default.
+    echo '{}' > /etc/docker/daemon.json
+    dockerd
 fi

diff --git a/tools/arvbox/lib/arvbox/docker/service/ready/run-service b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
index 3c7eff61765a8b638a9722cf6f489895e3d0c85f..6d5fe243e09d6ec639f166ba618001e94bc7f4d8 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/ready/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/ready/run-service
@@ -8,9 +8,6 @@
 set -eu -o pipefail
 
 if ! [[ -d /tmp/arvbox-ready ]] ; then
-   echo
-   echo "Arvados-in-a-box starting"
-   echo
    echo "Note: if this is a fresh arvbox installation, it may take 10-15 minutes (or longer) to download and"
    echo "install dependencies.  Use \"arvbox log\" to monitor the progress of specific services."
    echo

diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run b/tools/arvbox/lib/arvbox/docker/service/workbench/run
index a41922bb343948656d838f3d958c0b8131fa9d26..d90a2e2f8170d9d8d5960b69ba903822538a0ce0 100755 (executable)
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run
@@ -1,10 +1,12 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: AGPL-3.0
 
 set -e
 
+.  /usr/local/lib/arvbox/common.sh
+
 /usr/local/lib/arvbox/runsu.sh $0-service $1
 
 cd /usr/src/arvados/apps/workbench

diff --git a/vendor/vendor.json b/vendor/vendor.json
index 8551a06fb414573e90d2a802c2806c99e8bf0bd5..aeac93e475bb265d1ed7fb7fcfb75870604838e3 100644 (file)
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -24,23 +24,22 @@
                        "revisionTime": "2017-07-27T13:52:37Z"
                },
                {
-                       "checksumSHA1": "Rjy2uYZkQ8Kjht6ZFU0qzm2I/kI=",
-                       "origin": "github.com/docker/docker/vendor/github.com/Microsoft/go-winio",
+                       "checksumSHA1": "o/3cn04KAiwC7NqNVvmfVTD+hgA=",
                        "path": "github.com/Microsoft/go-winio",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "78439966b38d69bf38227fbf57ac8a6fee70f69a",
+                       "revisionTime": "2017-08-04T20:09:54Z"
                },
                {
-                       "checksumSHA1": "DYv6Q1+VfnUVxMwvk5IshAClLvw=",
+                       "checksumSHA1": "CWLxwFSj7MNed2MzAOSm0Cg9p+o=",
                        "path": "github.com/Sirupsen/logrus",
-                       "revision": "5e5dc898656f695e2a086b8e12559febbfc01562",
-                       "revisionTime": "2017-05-15T10:45:16Z"
+                       "revision": "d682213848ed68c0a260ca37d6dd5ace8423f5ba",
+                       "revisionTime": "2017-12-05T20:32:29Z"
                },
                {
                        "checksumSHA1": "+Zz+leZHHC9C0rx8DoRuffSRPso=",
                        "path": "github.com/coreos/go-systemd/daemon",
-                       "revision": "1f9909e51b2dab2487c26d64c8f2e7e580e4c9f5",
-                       "revisionTime": "2017-03-24T09:58:19Z"
+                       "revision": "cc4f39464dc797b91c8025330de585294c2a6950",
+                       "revisionTime": "2018-01-08T08:51:32Z"
                },
                {
                        "checksumSHA1": "pAu+do4x7E5SFLfIqJeGwhcOd6E=",
@@ -50,178 +49,141 @@
                },
                {
                        "checksumSHA1": "Gj+xR1VgFKKmFXYOJMnAczC3Znk=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/distribution/digestset",
                        "path": "github.com/docker/distribution/digestset",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "277ed486c948042cab91ad367c379524f3b25e18",
+                       "revisionTime": "2018-01-05T23:27:52Z"
                },
                {
-                       "checksumSHA1": "m4wEFD0Mh+ClfprUqgl0GyNmk7Q=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/distribution/reference",
+                       "checksumSHA1": "2Fe4D6PGaVE2he4fUeenLmhC1lE=",
                        "path": "github.com/docker/distribution/reference",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "277ed486c948042cab91ad367c379524f3b25e18",
+                       "revisionTime": "2018-01-05T23:27:52Z"
                },
                {
-                       "checksumSHA1": "5b7eC73lORtIUFCjz548jXkLlKU=",
+                       "checksumSHA1": "QKCQfrTv4wTL0KBDMHpWM/jHl9I=",
                        "path": "github.com/docker/docker/api",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "jFY7z3SSvQaJg51/nwzUkB/LDAc=",
+                       "checksumSHA1": "b91BIyJbqy05pXpEh1eGCJkdjYc=",
                        "path": "github.com/docker/docker/api/types",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "jVJDbe0IcyjoKc2xbohwzQr+FF0=",
                        "path": "github.com/docker/docker/api/types/blkiodev",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "AeSC0BOu1uapkGqfSXtfVSpwJzs=",
+                       "checksumSHA1": "DuOqFTQ95vKSuSE/Va88yRN/wb8=",
                        "path": "github.com/docker/docker/api/types/container",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "fzeGodcTcWuV18AT0BcvB4EFByo=",
+                       "checksumSHA1": "XDP7i6sMYGnUKeFzgt+mFBJwjjw=",
                        "path": "github.com/docker/docker/api/types/events",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "J2OKngfI3vgswudr9PZVUFcRRu0=",
+                       "checksumSHA1": "S4SWOa0XduRd8ene8Alwih2Nwcw=",
                        "path": "github.com/docker/docker/api/types/filters",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "yeB781yxPhnN6OXQ9/qSsyih3ek=",
+                       "checksumSHA1": "KuC0C6jo1t7tlvIqb7G3u1FIaZU=",
                        "path": "github.com/docker/docker/api/types/image",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "UK+VdM648oWzyqE4OqttgmPqjoA=",
+                       "checksumSHA1": "uJeLBKpHZXP+bWhXP4HhpyUTWYI=",
                        "path": "github.com/docker/docker/api/types/mount",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "oC2We2SzAYMULIfr7fXaoofnF+c=",
+                       "checksumSHA1": "Gskp+nvbVe8Gk1xPLHylZvNmqTg=",
                        "path": "github.com/docker/docker/api/types/network",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "r2vWq7Uc3ExKzMqYgH0b4AKjLKY=",
                        "path": "github.com/docker/docker/api/types/registry",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "VTxWyFud/RedrpllGdQonVtGM/A=",
                        "path": "github.com/docker/docker/api/types/strslice",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "B+yL7OD7dDV+n+XURK5x0N8U1pQ=",
+                       "checksumSHA1": "Q0U3queMsCw+rPPztXnRHwAxQEc=",
                        "path": "github.com/docker/docker/api/types/swarm",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "B7ZwKzrv3t3Vlox6/bYMHhMjsM8=",
+                       "checksumSHA1": "kVfD1e4Gak7k6tqDX5nrgQ57EYY=",
+                       "path": "github.com/docker/docker/api/types/swarm/runtime",
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
+               },
+               {
+                       "checksumSHA1": "77axKFOjRx1nGrzIggGXfTxUYVQ=",
                        "path": "github.com/docker/docker/api/types/time",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "uDPQ3nHsrvGQc9tg/J9OSC4N5dQ=",
                        "path": "github.com/docker/docker/api/types/versions",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "IBJy2zPEnYmcFJ3lM1eiRWnCxTA=",
                        "path": "github.com/docker/docker/api/types/volume",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "3bbQi817xzhn5andyThfuZcmHvk=",
+                       "checksumSHA1": "zQvx3WYTAwbPZEaVPjAsrmW7V00=",
                        "path": "github.com/docker/docker/client",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "jmo/t2zXAxirEPoFucNPXA/1SEc=",
-                       "path": "github.com/docker/docker/pkg/ioutils",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "ndnAFCfsGC3upNQ6jAEwzxcurww=",
-                       "path": "github.com/docker/docker/pkg/longpath",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "3u2xJnbqYSxOP3kOORetQD7P1Co=",
-                       "path": "github.com/docker/docker/pkg/mount",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "nhKmQDaWa+qKP2VWEy05CrTyids=",
-                       "path": "github.com/docker/docker/pkg/system",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "8I0Ez+aUYGpsDEVZ8wN/Ztf6Zqs=",
-                       "path": "github.com/docker/docker/pkg/tlsconfig",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "JbiWTzH699Sqz25XmDlsARpMN9w=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/go-connections/nat",
                        "path": "github.com/docker/go-connections/nat",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "3ede32e2033de7505e6500d6c868c2b9ed9f169d",
+                       "revisionTime": "2017-06-23T20:36:43Z"
                },
                {
                        "checksumSHA1": "jUfDG3VQsA2UZHvvIXncgiddpYA=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/go-connections/sockets",
                        "path": "github.com/docker/go-connections/sockets",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "3ede32e2033de7505e6500d6c868c2b9ed9f169d",
+                       "revisionTime": "2017-06-23T20:36:43Z"
                },
                {
-                       "checksumSHA1": "tuSzlS1UQ03+5Fvtqr5hI5sLLhA=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/go-connections/tlsconfig",
+                       "checksumSHA1": "c6lDGNwTm5mYq18IHP+lqYpk8xU=",
                        "path": "github.com/docker/go-connections/tlsconfig",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "3ede32e2033de7505e6500d6c868c2b9ed9f169d",
+                       "revisionTime": "2017-06-23T20:36:43Z"
                },
                {
-                       "checksumSHA1": "Pv1NbtDY4wETwwSD2DsI+s8de/I=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/go-units",
+                       "checksumSHA1": "kP4hqQGUNNXhgYxgB4AMWfNvmnA=",
                        "path": "github.com/docker/go-units",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
-               },
-               {
-                       "checksumSHA1": "r5hHPg1v9SL/gyatPWdaujG9aoE=",
-                       "origin": "github.com/docker/docker/vendor/github.com/docker/libtrust",
-                       "path": "github.com/docker/libtrust",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "d59758554a3d3911fa25c0269de1ebe2f1912c39",
+                       "revisionTime": "2017-12-21T20:03:56Z"
                },
                {
                        "checksumSHA1": "ImX1uv6O09ggFeBPUJJ2nu7MPSA=",
@@ -229,109 +191,279 @@
                        "revision": "0ca9ea5df5451ffdf184b4428c902747c2c11cd7",
                        "revisionTime": "2017-03-27T23:54:44Z"
                },
+               {
+                       "checksumSHA1": "wn2shNJMwRZpvuvkf1s7h0wvqHI=",
+                       "path": "github.com/gogo/protobuf/proto",
+                       "revision": "160de10b2537169b5ae3e7e221d28269ef40d311",
+                       "revisionTime": "2018-01-04T10:21:28Z"
+               },
                {
                        "checksumSHA1": "iIUYZyoanCQQTUaWsu8b+iOSPt4=",
                        "origin": "github.com/docker/docker/vendor/github.com/gorilla/context",
                        "path": "github.com/gorilla/context",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "zmCk+lgIeiOf0Ng9aFP9aFy8ksE=",
+                       "checksumSHA1": "fSs1WcPh2F5JJtxqYC+Jt8yCkYc=",
                        "path": "github.com/gorilla/mux",
-                       "revision": "4c1c3952b7d9d0a061a3fa7b36fd373ba0398ebc",
-                       "revisionTime": "2017-04-27T04:12:50Z"
+                       "revision": "5bbbb5b2b5729b132181cc7f4aa3b3c973e9a0ed",
+                       "revisionTime": "2018-01-07T15:57:08Z"
+               },
+               {
+                       "checksumSHA1": "d9PxF1XQGLMJZRct2R8qVM/eYlE=",
+                       "path": "github.com/hashicorp/golang-lru",
+                       "revision": "0a025b7e63adc15a622f29b0b2c4c3848243bbf6",
+                       "revisionTime": "2016-08-13T22:13:03Z"
+               },
+               {
+                       "checksumSHA1": "9hffs0bAIU6CquiRhKQdzjHnKt0=",
+                       "path": "github.com/hashicorp/golang-lru/simplelru",
+                       "revision": "0a025b7e63adc15a622f29b0b2c4c3848243bbf6",
+                       "revisionTime": "2016-08-13T22:13:03Z"
+               },
+               {
+                       "checksumSHA1": "iCsyavJDnXC9OY//p52IWJWy7PY=",
+                       "path": "github.com/jbenet/go-context/io",
+                       "revision": "d14ea06fba99483203c19d92cfcd13ebe73135f4",
+                       "revisionTime": "2015-07-11T00:45:18Z"
+               },
+               {
+                       "checksumSHA1": "oX6jFQD74oOApvDIhOzW2dXpg5Q=",
+                       "path": "github.com/kevinburke/ssh_config",
+                       "revision": "802051befeb51da415c46972b5caf36e7c33c53d",
+                       "revisionTime": "2017-10-13T21:14:58Z"
                },
                {
-                       "checksumSHA1": "uTUsjF7bymOuKvXbW2BpkK/w4Vg=",
+                       "checksumSHA1": "IfZcD4U1dtllJKlPNeD2aU4Jn98=",
                        "path": "github.com/lib/pq",
-                       "revision": "2704adc878c21e1329f46f6e56a1c387d788ff94",
-                       "revisionTime": "2017-03-24T20:46:54Z"
+                       "revision": "83612a56d3dd153a94a629cd64925371c9adad78",
+                       "revisionTime": "2017-11-26T05:04:59Z"
                },
                {
-                       "checksumSHA1": "q5SZBWFVC3wOIzftf+l/h5WLG1k=",
+                       "checksumSHA1": "AU3fA8Sm33Vj9PBoRPSeYfxLRuE=",
                        "path": "github.com/lib/pq/oid",
-                       "revision": "2704adc878c21e1329f46f6e56a1c387d788ff94",
-                       "revisionTime": "2017-03-24T20:46:54Z"
+                       "revision": "83612a56d3dd153a94a629cd64925371c9adad78",
+                       "revisionTime": "2017-11-26T05:04:59Z"
                },
                {
-                       "checksumSHA1": "OUupkCHnh8e3RHRNcMFqQJqhaQI=",
-                       "origin": "github.com/docker/docker/vendor/github.com/opencontainers/go-digest",
+                       "checksumSHA1": "V/quM7+em2ByJbWBLOsEwnY3j/Q=",
+                       "path": "github.com/mitchellh/go-homedir",
+                       "revision": "b8bc1bf767474819792c23f32d8286a45736f1c6",
+                       "revisionTime": "2016-12-03T19:45:07Z"
+               },
+               {
+                       "checksumSHA1": "OFNit1Qx2DdWhotfREKodDNUwCM=",
                        "path": "github.com/opencontainers/go-digest",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "279bed98673dd5bef374d3b6e4b09e2af76183bf",
+                       "revisionTime": "2017-06-07T19:53:33Z"
                },
                {
-                       "checksumSHA1": "qzsBSt0cnEYNIkYk7slHjmPl7o0=",
+                       "checksumSHA1": "ZGlIwSRjdLYCUII7JLE++N4w7Xc=",
                        "path": "github.com/opencontainers/image-spec/specs-go",
-                       "revision": "f03dbe35d449c54915d235f1a3cf8f585a24babe",
-                       "revisionTime": "2017-05-15T20:58:57Z"
+                       "revision": "577479e4dc273d3779f00c223c7e0dba4cd6b8b0",
+                       "revisionTime": "2017-11-25T02:40:18Z"
                },
                {
-                       "checksumSHA1": "vp5gfnUG920xBY1genCCoppZILc=",
+                       "checksumSHA1": "jdbXRRzeu0njLE9/nCEZG+Yg/Jk=",
                        "path": "github.com/opencontainers/image-spec/specs-go/v1",
-                       "revision": "f03dbe35d449c54915d235f1a3cf8f585a24babe",
-                       "revisionTime": "2017-05-15T20:58:57Z"
+                       "revision": "577479e4dc273d3779f00c223c7e0dba4cd6b8b0",
+                       "revisionTime": "2017-11-25T02:40:18Z"
                },
                {
-                       "checksumSHA1": "PdQm3s8DoVJ17Vk8n7o5iPa7PK0=",
-                       "origin": "github.com/docker/docker/vendor/github.com/pkg/errors",
+                       "checksumSHA1": "F1IYMLBLAZaTOWnmXsgaxTGvrWI=",
+                       "path": "github.com/pelletier/go-buffruneio",
+                       "revision": "c37440a7cf42ac63b919c752ca73a85067e05992",
+                       "revisionTime": "2017-02-27T22:03:11Z"
+               },
+               {
+                       "checksumSHA1": "xCv4GBFyw07vZkVtKF/XrUnkHRk=",
                        "path": "github.com/pkg/errors",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "e881fd58d78e04cf6d0de1217f8707c8cc2249bc",
+                       "revisionTime": "2017-12-16T07:03:16Z"
+               },
+               {
+                       "checksumSHA1": "UwtyqB7CaUWPlw0DVJQvw0IFQZs=",
+                       "path": "github.com/sergi/go-diff/diffmatchpatch",
+                       "revision": "1744e2970ca51c86172c8190fadad617561ed6e7",
+                       "revisionTime": "2017-11-10T11:01:46Z"
+               },
+               {
+                       "checksumSHA1": "8QeSG127zQqbA+YfkO1WkKx/iUI=",
+                       "path": "github.com/src-d/gcfg",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "yf5NBT8BofPfGYCXoLnj7BIA1wo=",
+                       "path": "github.com/src-d/gcfg/scanner",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "C5Z8YVyNTuvupM9AUr9KbPlps4Q=",
+                       "path": "github.com/src-d/gcfg/token",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "mDkN3UpR7auuFbwUuIwExz4DZgY=",
+                       "path": "github.com/src-d/gcfg/types",
+                       "revision": "f187355171c936ac84a82793659ebb4936bc1c23",
+                       "revisionTime": "2016-10-26T10:01:55Z"
+               },
+               {
+                       "checksumSHA1": "iHiMTBffQvWYlOLu3130JXuQpgQ=",
+                       "path": "github.com/xanzy/ssh-agent",
+                       "revision": "ba9c9e33906f58169366275e3450db66139a31a9",
+                       "revisionTime": "2015-12-15T15:34:51Z"
+               },
+               {
+                       "checksumSHA1": "TT1rac6kpQp2vz24m5yDGUNQ/QQ=",
+                       "path": "golang.org/x/crypto/cast5",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
                },
                {
-                       "checksumSHA1": "9jjO5GjLa0XF/nfWihF02RoH4qc=",
+                       "checksumSHA1": "IQkUIOnvlf0tYloFx9mLaXSvXWQ=",
+                       "path": "golang.org/x/crypto/curve25519",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "1hwn8cgg4EVXhCpJIqmMbzqnUo0=",
+                       "path": "golang.org/x/crypto/ed25519",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "LXFcVx8I587SnWmKycSDEq9yvK8=",
+                       "path": "golang.org/x/crypto/ed25519/internal/edwards25519",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "ooU7jaiYSUKlg5BVllI8lsq+5Qk=",
+                       "path": "golang.org/x/crypto/openpgp",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "olOKkhrdkYQHZ0lf1orrFQPQrv4=",
+                       "path": "golang.org/x/crypto/openpgp/armor",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "eo/KtdjieJQXH7Qy+faXFcF70ME=",
+                       "path": "golang.org/x/crypto/openpgp/elgamal",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "rlxVSaGgqdAgwblsErxTxIfuGfg=",
+                       "path": "golang.org/x/crypto/openpgp/errors",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "Pq88+Dgh04UdXWZN6P+bLgYnbRc=",
+                       "path": "golang.org/x/crypto/openpgp/packet",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "s2qT4UwvzBSkzXuiuMkowif1Olw=",
+                       "path": "golang.org/x/crypto/openpgp/s2k",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "NHjGg73p5iGZ+7tflJ4cVABNmKE=",
+                       "path": "golang.org/x/crypto/ssh",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "NMRX0onGReaL9IfLr0XQ3kl5Id0=",
+                       "path": "golang.org/x/crypto/ssh/agent",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "zBHtHvMj+MXa1qa4aglBt46uUck=",
+                       "path": "golang.org/x/crypto/ssh/knownhosts",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "X1NTlfcau2XcV6WtAHF6b/DECOA=",
+                       "path": "golang.org/x/crypto/ssh/terminal",
+                       "revision": "0fcca4842a8d74bfddc2c96a073bd2a4d2a7a2e8",
+                       "revisionTime": "2017-11-25T19:00:56Z"
+               },
+               {
+                       "checksumSHA1": "Y+HGqEkYM15ir+J93MEaHdyFy0c=",
                        "origin": "github.com/docker/docker/vendor/golang.org/x/net/context",
                        "path": "golang.org/x/net/context",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
                        "checksumSHA1": "WHc3uByvGaMcnSoI21fhzYgbOgg=",
-                       "origin": "github.com/docker/docker/vendor/golang.org/x/net/context/ctxhttp",
                        "path": "golang.org/x/net/context/ctxhttp",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "434ec0c7fe3742c984919a691b2018a6e9694425",
+                       "revisionTime": "2017-09-25T09:26:47Z"
                },
                {
-                       "checksumSHA1": "LvdVRE0FqdR68SvVpRkHs1rxhcA=",
-                       "origin": "github.com/docker/docker/vendor/golang.org/x/net/proxy",
+                       "checksumSHA1": "r9l4r3H6FOLQ0c2JaoXpopFjpnw=",
                        "path": "golang.org/x/net/proxy",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "434ec0c7fe3742c984919a691b2018a6e9694425",
+                       "revisionTime": "2017-09-25T09:26:47Z"
                },
                {
-                       "checksumSHA1": "yppNZB5y0GmJrt/TYOASrhe2oVc=",
+                       "checksumSHA1": "TBlnCuZUOzJHLu5DNY7XEj8TvbU=",
                        "path": "golang.org/x/net/webdav",
-                       "revision": "f01ecb60fe3835d80d9a0b7b2bf24b228c89260e",
-                       "revisionTime": "2017-07-11T11:58:19Z"
+                       "revision": "434ec0c7fe3742c984919a691b2018a6e9694425",
+                       "revisionTime": "2017-09-25T09:26:47Z"
                },
                {
                        "checksumSHA1": "XgtZlzd39qIkBHs6XYrq9dhTCog=",
                        "path": "golang.org/x/net/webdav/internal/xml",
-                       "revision": "f01ecb60fe3835d80d9a0b7b2bf24b228c89260e",
-                       "revisionTime": "2017-07-11T11:58:19Z"
+                       "revision": "434ec0c7fe3742c984919a691b2018a6e9694425",
+                       "revisionTime": "2017-09-25T09:26:47Z"
                },
                {
                        "checksumSHA1": "7EZyXN0EmZLgGxZxK01IJua4c8o=",
                        "path": "golang.org/x/net/websocket",
-                       "revision": "513929065c19401a1c7b76ecd942f9f86a0c061b",
-                       "revisionTime": "2017-05-12T22:20:15Z"
+                       "revision": "434ec0c7fe3742c984919a691b2018a6e9694425",
+                       "revisionTime": "2017-09-25T09:26:47Z"
                },
                {
-                       "checksumSHA1": "dUfdXzRJupI9VpqNR2LlppeZvLc=",
+                       "checksumSHA1": "znPq37/LZ4pJh7B4Lbu0ZuoMhNk=",
                        "origin": "github.com/docker/docker/vendor/golang.org/x/sys/unix",
                        "path": "golang.org/x/sys/unix",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
                },
                {
-                       "checksumSHA1": "fpW2dhGFC6SrVzipJx7fjg2DIH8=",
+                       "checksumSHA1": "8BcMOi8XTSigDtV2npDc8vMrS60=",
                        "origin": "github.com/docker/docker/vendor/golang.org/x/sys/windows",
                        "path": "golang.org/x/sys/windows",
-                       "revision": "280327cb4d1e1fe4f118d00596ce0b3a6ae6d07e",
-                       "revisionTime": "2017-05-17T20:48:28Z"
+                       "revision": "94b8a116fbf1cd90e68d8f5361b520d326a66f9b",
+                       "revisionTime": "2018-01-09T01:38:17Z"
+               },
+               {
+                       "checksumSHA1": "ziMb9+ANGRJSSIuxYdRbA+cDRBQ=",
+                       "path": "golang.org/x/text/transform",
+                       "revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
+                       "revisionTime": "2017-12-24T20:31:28Z"
+               },
+               {
+                       "checksumSHA1": "BCNYmf4Ek93G4lk5x3ucNi/lTwA=",
+                       "path": "golang.org/x/text/unicode/norm",
+                       "revision": "e19ae1496984b1c655b8044a65c0300a3c878dd3",
+                       "revisionTime": "2017-12-24T20:31:28Z"
                },
                {
                        "checksumSHA1": "CEFTYXtWmgSh+3Ik1NmDaJcz4E0=",
@@ -340,10 +472,292 @@
                        "revisionTime": "2016-12-08T18:13:25Z"
                },
                {
-                       "checksumSHA1": "fALlQNY1fM99NesfLJ50KguWsio=",
+                       "checksumSHA1": "GdsHg+yOsZtdMvD9HJFovPsqKec=",
+                       "path": "gopkg.in/src-d/go-billy.v4",
+                       "revision": "053dbd006f81a230434f712314aacfb540b52cc5",
+                       "revisionTime": "2017-11-27T19:20:57Z"
+               },
+               {
+                       "checksumSHA1": "yscejfasrttJfPq91pn7gArFb5o=",
+                       "path": "gopkg.in/src-d/go-billy.v4/helper/chroot",
+                       "revision": "053dbd006f81a230434f712314aacfb540b52cc5",
+                       "revisionTime": "2017-11-27T19:20:57Z"
+               },
+               {
+                       "checksumSHA1": "B7HAyGfl+ONIAvlHzbvSsLisx9o=",
+                       "path": "gopkg.in/src-d/go-billy.v4/helper/polyfill",
+                       "revision": "053dbd006f81a230434f712314aacfb540b52cc5",
+                       "revisionTime": "2017-11-27T19:20:57Z"
+               },
+               {
+                       "checksumSHA1": "1CnG3JdmIQoa6mE0O98BfymLmuM=",
+                       "path": "gopkg.in/src-d/go-billy.v4/osfs",
+                       "revision": "053dbd006f81a230434f712314aacfb540b52cc5",
+                       "revisionTime": "2017-11-27T19:20:57Z"
+               },
+               {
+                       "checksumSHA1": "lo42NuhQJppy2ne/uwPR2T9BSPY=",
+                       "path": "gopkg.in/src-d/go-billy.v4/util",
+                       "revision": "053dbd006f81a230434f712314aacfb540b52cc5",
+                       "revisionTime": "2017-11-27T19:20:57Z"
+               },
+               {
+                       "checksumSHA1": "ydjzL2seh3M8h9svrSDV5y/KQJU=",
+                       "path": "gopkg.in/src-d/go-git.v4",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "TSoIlaADKlw3Zx0ysCCBn6kyXNE=",
+                       "path": "gopkg.in/src-d/go-git.v4/config",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "B2OLPJ4wnJIM2TMjTyzusYluUeI=",
+                       "path": "gopkg.in/src-d/go-git.v4/internal/revision",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "o9YH41kQMefVGUS7d3WWSLLhIRk=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "BrsKLhmB0BtaMY+ol1oglnHhvrs=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/cache",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "pHPMiAzXG/TJqTLEKj2SHjxX4zs=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/filemode",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "UGIM9BX7w3MhiadsuN6f8Bx0VZU=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/config",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "L1H7nPf65//6nQGt3Lzq16vLD8w=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/diff",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "87WhYdropmGA4peZOembY5hEgq8=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/gitignore",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "G0TX3efLdk7noo/n1Dt9Tzempig=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/idxfile",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "q7HtzrSzVE9qN5N3QOxkLFcZI1U=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/index",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "0IxJpGMfdnr3cuuVE59u+1B5n9o=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/objfile",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "LJnyldAM69WmMXW5avaEeSScKTU=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/packfile",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "T8efjPxCKp23RvSBI51qugHzgxw=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/format/pktline",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "97LEL3gxgDWPP/UlRHMfKb5I0RA=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/object",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "PQmY1mHiPdNBNrh3lESZe3QH36c=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "JjHHYoWDYf0H//nP2FIS05ZLgj8=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/capability",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "wVfbzV5BNhjW/HFFJuTCjkPSJ5M=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/protocol/packp/sideband",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "m8nTTRFD7kmX9nT5Yfr9lqabR4s=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/revlist",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "Xito+BwVCMpKrhcvgz5wU+MRmEo=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/storer",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "AVSX04sTj3cBv1muAmIbPE9D9FY=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "cmOntUALmiRvvblEXAQXNO4Oous=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/client",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "gaKy+c/OjPQFLhENnSAFEZUngok=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/file",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "chcAwbm6J5uXXn6IV58+G6RKCjU=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/git",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "m9TNeIIGUBdZ0qdSl5Xa/0TIvfo=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/http",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "6asrmcjb98FpRr83ICCODXdGWdE=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/internal/common",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "MGiWWrsy8iQ5ZdCXEN2Oc4oprCk=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/server",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "vat8YhxXGXNcg8HvCDfHAR6BcL0=",
+                       "path": "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "FlVLBdu4cjlXj9zjRRNDurRLABU=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "IpSxC31PynwJBajOaHR7gtnVc7I=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/filesystem",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "OaZO6dgvn6PMvezw0bYQUGLSrF0=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/filesystem/internal/dotgit",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "jPRm9YqpcJzx4oasd6PBdD33Dgo=",
+                       "path": "gopkg.in/src-d/go-git.v4/storage/memory",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "AzdUpuGqSNnNK6DgdNjWrn99i3o=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/binary",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "vniUxB6bbDYazl21cOfmhdZZiY8=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/diff",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "cspCXRxvzvoNOEUB7wRgOKYrVjQ=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/ioutil",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "shsY2I1OFbnjopNWF21Tkfx+tac=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "QiHHx1Qb/Vv4W6uQb+mJU2zMqLo=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/filesystem",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "M+6y9mdBFksksEGBceBh9Se3W7Y=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/index",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "7eEw/xsSrFLfSppRf/JIt9u7lbU=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/internal/frame",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "qCb9d3cwnPHVLqS/U9NAzK+1Ptg=",
+                       "path": "gopkg.in/src-d/go-git.v4/utils/merkletrie/noder",
+                       "revision": "bf3b1f1fb9e0a04d0f87511a7ded2562b48a19d8",
+                       "revisionTime": "2018-01-08T13:05:52Z"
+               },
+               {
+                       "checksumSHA1": "I4c3qsEX8KAUTeB9+2pwVX/2ojU=",
+                       "path": "gopkg.in/warnings.v0",
+                       "revision": "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b",
+                       "revisionTime": "2017-11-15T19:30:34Z"
+               },
+               {
+                       "checksumSHA1": "qOmvuDm+F+2nQQecUZBVkZrTn6Y=",
                        "path": "gopkg.in/yaml.v2",
-                       "revision": "cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b",
-                       "revisionTime": "2017-04-07T17:21:22Z"
+                       "revision": "d670f9405373e636a5a2765eea47fac0c9bc91a4",
+                       "revisionTime": "2018-01-09T11:43:31Z"
+               },
+               {
+                       "checksumSHA1": "rBIcwbUjE9w1aV0qh7lAL1hcxCQ=",
+                       "path": "rsc.io/getopt",
+                       "revision": "20be20937449f18bb9967c10d732849fb4401e63",
+                       "revisionTime": "2017-08-11T00:05:52Z"
                }
        ],
        "rootPath": "git.curoverse.com/arvados.git"