14299: Use include_old_versions in keep-balance to avoid losing blocks.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>

diff --git a/sdk/go/arvados/resource_list.go b/sdk/go/arvados/resource_list.go
index e4b5f65408447fba417ad12fd99d675c5601cddd..14ce098cfc1a54f0f0de74aa9cf60ca8274a693e 100644 (file)
--- a/sdk/go/arvados/resource_list.go
+++ b/sdk/go/arvados/resource_list.go
@@ -9,14 +9,15 @@ import "encoding/json"
 // ResourceListParams expresses which results are requested in a
 // list/index API.
 type ResourceListParams struct {
-       Select       []string `json:"select,omitempty"`
-       Filters      []Filter `json:"filters,omitempty"`
-       IncludeTrash bool     `json:"include_trash,omitempty"`
-       Limit        *int     `json:"limit,omitempty"`
-       Offset       int      `json:"offset,omitempty"`
-       Order        string   `json:"order,omitempty"`
-       Distinct     bool     `json:"distinct,omitempty"`
-       Count        string   `json:"count,omitempty"`
+       Select             []string `json:"select,omitempty"`
+       Filters            []Filter `json:"filters,omitempty"`
+       IncludeTrash       bool     `json:"include_trash,omitempty"`
+       IncludeOldVersions bool     `json:"include_old_versions,omitempty"`
+       Limit              *int     `json:"limit,omitempty"`
+       Offset             int      `json:"offset,omitempty"`
+       Order              string   `json:"order,omitempty"`
+       Distinct           bool     `json:"distinct,omitempty"`
+       Count              string   `json:"count,omitempty"`
 }
 
 // A Filter restricts the set of records returned by a list/index API.

diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 923ea47c429a6051377575376860b0bf51fa5915..f7cb7f92bd71d2ea248c8febbd8978e4c01d0c38 100644 (file)
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -17,7 +17,6 @@ import (
 
        "git.curoverse.com/arvados.git/sdk/go/arvados"
        "github.com/Sirupsen/logrus"
-
        check "gopkg.in/check.v1"
 )
 
@@ -424,6 +423,7 @@ func (s *runSuite) TestDryRun(c *check.C) {
        c.Check(err, check.IsNil)
        for _, req := range collReqs.reqs {
                c.Check(req.Form.Get("include_trash"), check.Equals, "true")
+               c.Check(req.Form.Get("include_old_versions"), check.Equals, "true")
        }
        c.Check(trashReqs.Count(), check.Equals, 0)
        c.Check(pullReqs.Count(), check.Equals, 0)

diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index 8f4ebb6bdfa277b33deeaf6cf2c0e2f2ecade076..1e5fa5797855048bcb9db80487d7f6a8e4486787 100644 (file)
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -36,7 +36,8 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
        }
 
        expectCount, err := countCollections(c, arvados.ResourceListParams{
-               IncludeTrash: true,
+               IncludeTrash:       true,
+               IncludeOldVersions: true,
        })
        if err != nil {
                return err
@@ -48,11 +49,12 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
                limit = 1<<31 - 1
        }
        params := arvados.ResourceListParams{
-               Limit:        &limit,
-               Order:        "modified_at, uuid",
-               Count:        "none",
-               Select:       []string{"uuid", "unsigned_manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
-               IncludeTrash: true,
+               Limit:              &limit,
+               Order:              "modified_at, uuid",
+               Count:              "none",
+               Select:             []string{"uuid", "unsigned_manifest_text", "modified_at", "portable_data_hash", "replication_desired"},
+               IncludeTrash:       true,
+               IncludeOldVersions: true,
        }
        var last arvados.Collection
        var filterTime time.Time
@@ -140,7 +142,8 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
                        Attr:     "modified_at",
                        Operator: "<=",
                        Operand:  filterTime}},
-               IncludeTrash: true,
+               IncludeTrash:       true,
+               IncludeOldVersions: true,
        }); err != nil {
                return err
        } else if callCount < checkCount {

diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index 5280b40c9123012210010585308dd883e3d7962a..ceffb9cc1f5c7b78874f4cf4bc0d847a8f40d46c 100644 (file)
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -16,7 +16,6 @@ import (
        "git.curoverse.com/arvados.git/sdk/go/arvadostest"
        "git.curoverse.com/arvados.git/sdk/go/keepclient"
        "github.com/Sirupsen/logrus"
-
        check "gopkg.in/check.v1"
 )