projects / arvados.git / blobdiff
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
16535: Merge branch 'master'
[arvados.git] / lib / recovercollection / cmd.go
diff --git a/lib/recovercollection/cmd.go b/lib/recovercollection/cmd.go
index d19bf311680814fa612b2ac6f22a56b4d026e4e7..da466c31ca7d2a3a4a13f23fc92177d04ef087ec 100644 (file)
--- a/lib/recovercollection/cmd.go
+++ b/lib/recovercollection/cmd.go
@@ -42,7 +42,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `Usage:
        flags.SetOutput(stderr)
        flags.Usage = func() {
                fmt.Fprintf(flags.Output(), `Usage:
-       %s [options ...] { /path/to/manifest.txt | log-entry-uuid } [...]
+       %s [options ...] { /path/to/manifest.txt | log-or-collection-uuid } [...]
 
        This program recovers deleted collections. Recovery is
        possible when the collection's manifest is still available and
 
        This program recovers deleted collections. Recovery is
        possible when the collection's manifest is still available and
@@ -52,10 +52,18 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
        collections, or the blocks have been trashed but not yet
        deleted).
 
        collections, or the blocks have been trashed but not yet
        deleted).
 
-       Collections can be specified either by filename (a local file
-       containing a manifest with the desired data) or by log UUID
-       (an Arvados log entry, typically a "delete" or "update" event,
-       whose "old attributes" have a manifest with the desired data).
+       There are multiple ways to specify a collection to recover:
+
+        * Path to a local file containing a manifest with the desired
+         data
+
+       * UUID of an Arvados log entry, typically a "delete" or
+         "update" event, whose "old attributes" have a manifest with
+         the desired data
+
+       * UUID of an Arvados collection whose most recent log entry,
+          typically a "delete" or "update" event, has the desired
+          data in its "old attributes"
 
        For each provided collection manifest, once all data blocks
        are recovered/protected from garbage collection, a new
 
        For each provided collection manifest, once all data blocks
        are recovered/protected from garbage collection, a new
@@ -113,29 +121,58 @@ Options:
        for _, src := range flags.Args() {
                logger := logger.WithField("src", src)
                var mtxt string
        for _, src := range flags.Args() {
                logger := logger.WithField("src", src)
                var mtxt string
-               if len(src) == 27 && src[5:12] == "-57u5n-" {
-                       var logent struct {
-                               EventType  string    `json:"event_type"`
-                               EventAt    time.Time `json:"event_at"`
-                               ObjectUUID string    `json:"object_uuid"`
-                               Properties struct {
-                                       OldAttributes struct {
-                                               ManifestText string `json:"manifest_text"`
-                                       } `json:"old_attributes"`
-                               } `json:"properties"`
+               if !strings.Contains(src, "/") && len(src) == 27 && src[5] == '-' && src[11] == '-' {
+                       var filters []arvados.Filter
+                       if src[5:12] == "-57u5n-" {
+                               filters = []arvados.Filter{{"uuid", "=", src}}
+                       } else if src[5:12] == "-4zz18-" {
+                               filters = []arvados.Filter{{"object_uuid", "=", src}}
+                       } else {
+                               logger.Error("looks like a UUID but not a log or collection UUID (if it's really a file, prepend './')")
+                               exitcode = 1
+                               continue
+                       }
+                       var resp struct {
+                               Items []struct {
+                                       UUID       string    `json:"uuid"`
+                                       EventType  string    `json:"event_type"`
+                                       EventAt    time.Time `json:"event_at"`
+                                       ObjectUUID string    `json:"object_uuid"`
+                                       Properties struct {
+                                               OldAttributes struct {
+                                                       ManifestText string `json:"manifest_text"`
+                                               } `json:"old_attributes"`
+                                       } `json:"properties"`
+                               }
                        }
                        }
-                       err = client.RequestAndDecode(&logent, "GET", "arvados/v1/logs/"+src, nil, nil)
+                       err = client.RequestAndDecode(&resp, "GET", "arvados/v1/logs", nil, arvados.ListOptions{
+                               Limit:   1,
+                               Order:   []string{"event_at desc"},
+                               Filters: filters,
+                       })
                        if err != nil {
                        if err != nil {
-                               logger.WithError(err).Error("failed to load log entry")
+                               logger.WithError(err).Error("error looking up log entry")
+                               exitcode = 1
+                               continue
+                       } else if len(resp.Items) == 0 {
+                               logger.Error("log entry not found")
                                exitcode = 1
                                continue
                        }
                                exitcode = 1
                                continue
                        }
+                       logent := resp.Items[0]
                        logger.WithFields(logrus.Fields{
                        logger.WithFields(logrus.Fields{
+                               "uuid":                logent.UUID,
                                "old_collection_uuid": logent.ObjectUUID,
                                "logged_event_type":   logent.EventType,
                                "logged_event_time":   logent.EventAt,
                                "old_collection_uuid": logent.ObjectUUID,
                                "logged_event_type":   logent.EventType,
                                "logged_event_time":   logent.EventAt,
+                               "logged_object_uuid":  logent.ObjectUUID,
                        }).Info("loaded log entry")
                        mtxt = logent.Properties.OldAttributes.ManifestText
                        }).Info("loaded log entry")
                        mtxt = logent.Properties.OldAttributes.ManifestText
+                       if mtxt == "" {
+                               logger.Error("log entry properties.old_attributes.manifest_text missing or empty")
+                               exitcode = 1
+                               continue
+                       }
                } else {
                        buf, err := ioutil.ReadFile(src)
                        if err != nil {
                } else {
                        buf, err := ioutil.ReadFile(src)
                        if err != nil {
@@ -167,8 +204,8 @@ var errNotFound = errors.New("not found")
 
 // Finds the timestamp of the newest copy of blk on svc. Returns
 // errNotFound if blk is not on svc at all.
 
 // Finds the timestamp of the newest copy of blk on svc. Returns
 // errNotFound if blk is not on svc at all.
-func (rcvr recoverer) newestMtime(logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
-       found, err := svc.Index(rcvr.client, blk)
+func (rcvr recoverer) newestMtime(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+       found, err := svc.Index(ctx, rcvr.client, blk)
        if err != nil {
                logger.WithError(err).Warn("error getting index")
                return time.Time{}, err
        if err != nil {
                logger.WithError(err).Warn("error getting index")
                return time.Time{}, err
@@ -199,7 +236,7 @@ var errTouchIneffective = errors.New("(BUG?) touch succeeded but had no effect -
 // saved. But if the block's timestamp is more recent than blobsigttl,
 // keepstore will refuse to trash it even if told to by keep-balance.
 func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
 // saved. But if the block's timestamp is more recent than blobsigttl,
 // keepstore will refuse to trash it even if told to by keep-balance.
 func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
-       if latest, err := rcvr.newestMtime(logger, blk, svc); err != nil {
+       if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err != nil {
                return err
        } else if latest.Add(blobsigttl).After(blobsigexp) {
                return nil
                return err
        } else if latest.Add(blobsigttl).After(blobsigexp) {
                return nil
@@ -208,7 +245,7 @@ func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger,
                return fmt.Errorf("error updating timestamp: %s", err)
        }
        logger.Debug("updated timestamp")
                return fmt.Errorf("error updating timestamp: %s", err)
        }
        logger.Debug("updated timestamp")
-       if latest, err := rcvr.newestMtime(logger, blk, svc); err == errNotFound {
+       if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err == errNotFound {
                return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
        } else if err != nil {
                return err
                return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
        } else if err != nil {
                return err