16427: Option to recover from given collection's last log entry.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --git a/lib/recovercollection/cmd.go b/lib/recovercollection/cmd.go
index d19bf311680814fa612b2ac6f22a56b4d026e4e7..cea4607c98fe533fec8b37f839f1f641ce3fcccb 100644 (file)
--- a/lib/recovercollection/cmd.go
+++ b/lib/recovercollection/cmd.go
@@ -42,7 +42,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `Usage:
-       %s [options ...] { /path/to/manifest.txt | log-entry-uuid } [...]
+       %s [options ...] { /path/to/manifest.txt | log-or-collection-uuid } [...]
 
        This program recovers deleted collections. Recovery is
        possible when the collection's manifest is still available and
@@ -52,10 +52,18 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        collections, or the blocks have been trashed but not yet
        deleted).
 
-       Collections can be specified either by filename (a local file
-       containing a manifest with the desired data) or by log UUID
-       (an Arvados log entry, typically a "delete" or "update" event,
-       whose "old attributes" have a manifest with the desired data).
+       There are multiple ways to specify a collection to recover:
+
+        * Path to a local file containing a manifest with the desired
+         data
+
+       * UUID of an Arvados log entry, typically a "delete" or
+         "update" event, whose "old attributes" have a manifest with
+         the desired data
+
+       * UUID of an Arvados collection whose most recent log entry,
+          typically a "delete" or "update" event, has the desired
+          data in its "old attributes"
 
        For each provided collection manifest, once all data blocks
        are recovered/protected from garbage collection, a new
@@ -113,29 +121,58 @@ Options:
        for _, src := range flags.Args() {
                logger := logger.WithField("src", src)
                var mtxt string
-               if len(src) == 27 && src[5:12] == "-57u5n-" {
-                       var logent struct {
-                               EventType  string    `json:"event_type"`
-                               EventAt    time.Time `json:"event_at"`
-                               ObjectUUID string    `json:"object_uuid"`
-                               Properties struct {
-                                       OldAttributes struct {
-                                               ManifestText string `json:"manifest_text"`
-                                       } `json:"old_attributes"`
-                               } `json:"properties"`
+               if !strings.Contains(src, "/") && len(src) == 27 && src[5] == '-' && src[11] == '-' {
+                       var filters []arvados.Filter
+                       if src[5:12] == "-57u5n-" {
+                               filters = []arvados.Filter{{"uuid", "=", src}}
+                       } else if src[5:12] == "-4zz18-" {
+                               filters = []arvados.Filter{{"object_uuid", "=", src}}
+                       } else {
+                               logger.Error("looks like a UUID but not a log or collection UUID (if it's really a file, prepend './')")
+                               exitcode = 1
+                               continue
+                       }
+                       var resp struct {
+                               Items []struct {
+                                       UUID       string    `json:"uuid"`
+                                       EventType  string    `json:"event_type"`
+                                       EventAt    time.Time `json:"event_at"`
+                                       ObjectUUID string    `json:"object_uuid"`
+                                       Properties struct {
+                                               OldAttributes struct {
+                                                       ManifestText string `json:"manifest_text"`
+                                               } `json:"old_attributes"`
+                                       } `json:"properties"`
+                               }
                        }
-                       err = client.RequestAndDecode(&logent, "GET", "arvados/v1/logs/"+src, nil, nil)
+                       err = client.RequestAndDecode(&resp, "GET", "arvados/v1/logs", nil, arvados.ListOptions{
+                               Limit:   1,
+                               Order:   []string{"event_at desc"},
+                               Filters: filters,
+                       })
                        if err != nil {
-                               logger.WithError(err).Error("failed to load log entry")
+                               logger.WithError(err).Error("error looking up log entry")
+                               exitcode = 1
+                               continue
+                       } else if len(resp.Items) == 0 {
+                               logger.Error("log entry not found")
                                exitcode = 1
                                continue
                        }
+                       logent := resp.Items[0]
                        logger.WithFields(logrus.Fields{
+                               "uuid":                logent.UUID,
                                "old_collection_uuid": logent.ObjectUUID,
                                "logged_event_type":   logent.EventType,
                                "logged_event_time":   logent.EventAt,
+                               "logged_object_uuid":  logent.ObjectUUID,
                        }).Info("loaded log entry")
                        mtxt = logent.Properties.OldAttributes.ManifestText
+                       if mtxt == "" {
+                               logger.Error("log entry properties.old_attributes.manifest_text missing or empty")
+                               exitcode = 1
+                               continue
+                       }
                } else {
                        buf, err := ioutil.ReadFile(src)
                        if err != nil {

diff --git a/lib/recovercollection/cmd_test.go b/lib/recovercollection/cmd_test.go
index a6bf19de2db361f5b8425e914d655ef91160368b..57c2c64cdab01289911043c5767ed92edbcd8c36 100644 (file)
--- a/lib/recovercollection/cmd_test.go
+++ b/lib/recovercollection/cmd_test.go
@@ -36,7 +36,7 @@ func (*Suite) TestUnrecoverableBlock(c *check.C) {
        mfile := tmp + "/manifest"
        ioutil.WriteFile(mfile, []byte(". aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+410 0:410:Gone\n"), 0777)
        var stdout, stderr bytes.Buffer
-       exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+       exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
        c.Check(exitcode, check.Equals, 1)
        c.Check(stdout.String(), check.Equals, "")
        c.Log(stderr.String())
@@ -93,7 +93,7 @@ func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
        }
 
        var stdout, stderr bytes.Buffer
-       exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+       exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
        c.Check(exitcode, check.Equals, 0)
        c.Check(stdout.String(), check.Matches, `zzzzz-4zz18-.{15}\n`)
        c.Log(stderr.String())
@@ -115,3 +115,22 @@ func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
        }
        c.Check(found, check.Equals, true)
 }
+
+func (*Suite) TestUnusableManifestSourceArg(c *check.C) {
+       for _, trial := range []struct {
+               srcArg    string
+               errRegexp string
+       }{
+               {"zzzzz-4zz18-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found".*`},
+               {"zzzzz-57u5n-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found.*`},
+               {"zzzzz-57u5n-containerlog006", `(?ms).*msg="log entry properties\.old_attributes\.manifest_text missing or empty".*`},
+               {"zzzzz-j7d0g-aaaaaaaaaaaaaaa", `(?ms).*msg="looks like a UUID but not a log or collection UUID.*`},
+       } {
+               var stdout, stderr bytes.Buffer
+               exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", trial.srcArg}, &bytes.Buffer{}, &stdout, &stderr)
+               c.Check(exitcode, check.Equals, 1)
+               c.Check(stdout.String(), check.Equals, "")
+               c.Log(stderr.String())
+               c.Check(stderr.String(), check.Matches, trial.errRegexp)
+       }
+}