9437: Report timestamps as nanoseconds since epoch in keepstore index.
authorTom Clegg <tom@curoverse.com>
Thu, 23 Jun 2016 20:19:30 +0000 (16:19 -0400)
committerTom Clegg <tom@curoverse.com>
Fri, 1 Jul 2016 17:53:12 +0000 (13:53 -0400)
sdk/go/arvados/keep_service.go
services/datamanager/summary/trash_list.go
services/keep-balance/balance.go
services/keep-balance/balance_test.go
services/keepstore/azure_blob_volume.go
services/keepstore/s3_volume.go
services/keepstore/trash_worker.go
services/keepstore/trash_worker_test.go
services/keepstore/volume_generic_test.go
services/keepstore/volume_unix.go

index 4af1b7910f6f3b111583ad91fa5416ef520b4ac5..87dbd2a9e9e623385fd2f52c3266de80a9a0b345 100644 (file)
@@ -30,6 +30,7 @@ type KeepServiceList struct {
 // us about a stored block.
 type KeepServiceIndexEntry struct {
        SizedDigest
+       // Time of last write, in nanoseconds since Unix epoch
        Mtime int64
 }
 
index b6ceacecde2b8e2ffe810deea9e3777aade06625..3e4d387b62e2c4ba3c7d039a7114bd5ad222d2da 100644 (file)
@@ -29,7 +29,7 @@ func BuildTrashLists(kc *keepclient.KeepClient,
        ttl := int64(_ttl.(float64))
 
        // expire unreferenced blocks more than "ttl" seconds old.
-       expiry := time.Now().UTC().Unix() - ttl
+       expiry := time.Now().UTC().UnixNano() - ttl*1e9
 
        return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil
 }
index 2d1a59e8909bc250b2ca995775496f1839adf9f9..fa585d3aeb90e089e16ae041e0a3955f725b3f24 100644 (file)
@@ -199,7 +199,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
                return err
        }
        bal.DefaultReplication = dd.DefaultCollectionReplication
-       bal.MinMtime = time.Now().Unix() - dd.BlobSignatureTTL
+       bal.MinMtime = time.Now().UnixNano() - dd.BlobSignatureTTL*1e9
 
        errs := make(chan error, 2+len(bal.KeepServices))
        wg := sync.WaitGroup{}
index 682a5fb070cf0ab7e8a2b0fff0bb92a750622e06..b93939c0526d3c1f8bb7da93fe1f9915ad74c6cc 100644 (file)
@@ -76,7 +76,7 @@ func (bal *balancerSuite) SetUpTest(c *check.C) {
                bal.KeepServices[srv.UUID] = srv
        }
 
-       bal.MinMtime = time.Now().Unix() - bal.signatureTTL
+       bal.MinMtime = time.Now().UnixNano() - bal.signatureTTL*1e9
 }
 
 func (bal *balancerSuite) TestPerfect(c *check.C) {
@@ -240,7 +240,7 @@ func (bal *balancerSuite) srvList(knownBlockID int, order slots) (srvs []*KeepSe
 // replList is like srvList but returns an "existing replicas" slice,
 // suitable for a BlockState test fixture.
 func (bal *balancerSuite) replList(knownBlockID int, order slots) (repls []Replica) {
-       mtime := time.Now().Unix() - bal.signatureTTL - 86400
+       mtime := time.Now().UnixNano() - (bal.signatureTTL+86400)*1e9
        for _, srv := range bal.srvList(knownBlockID, order) {
                repls = append(repls, Replica{srv, mtime})
                mtime++
index 99da2a3a3de35de90be820b1a5285e5b592004d7..48cb02647cfd098cdc67796ba992ac5cba327bde 100644 (file)
@@ -350,7 +350,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
                                // Trashed blob; exclude it from response
                                continue
                        }
-                       fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.Unix())
+                       fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.UnixNano())
                }
                if resp.NextMarker == "" {
                        return nil
index 80a7c89f2ed4f6669566711c40c4d0a59940e439..b1b198d02de6dc0c158dfacdade39c1d9de53a16 100644 (file)
@@ -249,7 +249,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
                        if !v.isKeepBlock(key.Key) {
                                continue
                        }
-                       fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.Unix())
+                       fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.UnixNano())
                }
                if !listResp.IsTruncated {
                        break
index 62f63d57c8edb655b5078ebf637ce6d0ed0475bb..d11bc05192246a75e8ba4c95bd544b0712279ff6 100644 (file)
@@ -22,7 +22,7 @@ func RunTrashWorker(trashq *WorkQueue) {
 
 // TrashItem deletes the indicated block from every writable volume.
 func TrashItem(trashRequest TrashRequest) {
-       reqMtime := time.Unix(trashRequest.BlockMtime, 0)
+       reqMtime := time.Unix(0, trashRequest.BlockMtime)
        if time.Since(reqMtime) < blobSignatureTTL {
                log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
                        time.Since(reqMtime),
@@ -39,8 +39,8 @@ func TrashItem(trashRequest TrashRequest) {
                        log.Printf("%v Delete(%v): %v", volume, trashRequest.Locator, err)
                        continue
                }
-               if trashRequest.BlockMtime != mtime.Unix() {
-                       log.Printf("%v Delete(%v): mtime on volume is %v does not match trash list value %v", volume, trashRequest.Locator, mtime.Unix(), trashRequest.BlockMtime)
+               if trashRequest.BlockMtime != mtime.UnixNano() {
+                       log.Printf("%v Delete(%v): stored mtime %v does not match trash list value %v", volume, trashRequest.Locator, mtime.UnixNano(), trashRequest.BlockMtime)
                        continue
                }
 
index d111caeac8e5b571202502e0aea63f07816365ba..94798d95acfd85216ad60982b71282d84530ef7d 100644 (file)
@@ -236,7 +236,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
        // Create TrashRequest for the test
        trashRequest := TrashRequest{
                Locator:    testData.DeleteLocator,
-               BlockMtime: oldBlockTime.Unix(),
+               BlockMtime: oldBlockTime.UnixNano(),
        }
 
        // Run trash worker and put the trashRequest on trashq
index f8fe0d0ebce719c6c823fe9caa9fcce12324eb49..4291c6cd1f3964f06a095214d2e7308d35ea93d4 100644 (file)
@@ -7,6 +7,7 @@ import (
        "os"
        "regexp"
        "sort"
+       "strconv"
        "strings"
        "time"
 
@@ -355,10 +356,22 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
        v := factory(t)
        defer v.Teardown()
 
+       // minMtime and maxMtime are the minimum and maximum
+       // acceptable values the index can report for our test
+       // blocks. 1-second precision is acceptable.
+       minMtime := time.Now().UTC().UnixNano()
+       minMtime -= minMtime % 1e9
+
        v.PutRaw(TestHash, TestBlock)
        v.PutRaw(TestHash2, TestBlock2)
        v.PutRaw(TestHash3, TestBlock3)
 
+       maxMtime := time.Now().UTC().UnixNano()
+       if maxMtime%1e9 > 0 {
+               maxMtime -= maxMtime % 1e9
+               maxMtime += 1e9
+       }
+
        // Blocks whose names aren't Keep hashes should be omitted from
        // index
        v.PutRaw("fffffffffnotreallyahashfffffffff", nil)
@@ -371,15 +384,21 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
        indexRows := strings.Split(string(buf.Bytes()), "\n")
        sort.Strings(indexRows)
        sortedIndex := strings.Join(indexRows, "\n")
-       m, err := regexp.MatchString(
-               `^\n`+TestHash+`\+\d+ \d+\n`+
-                       TestHash3+`\+\d+ \d+\n`+
-                       TestHash2+`\+\d+ \d+$`,
-               sortedIndex)
-       if err != nil {
-               t.Error(err)
-       } else if !m {
+       m := regexp.MustCompile(
+               `^\n` + TestHash + `\+\d+ (\d+)\n` +
+                       TestHash3 + `\+\d+ \d+\n` +
+                       TestHash2 + `\+\d+ \d+$`,
+       ).FindStringSubmatch(sortedIndex)
+       if m == nil {
                t.Errorf("Got index %q for empty prefix", sortedIndex)
+       } else {
+               mtime, err := strconv.ParseInt(m[1], 10, 64)
+               if err != nil {
+                       t.Error(err)
+               } else if mtime < minMtime || mtime > maxMtime {
+                       t.Errorf("got %d for TestHash timestamp, expected %d <= t <= %d",
+                               mtime, minMtime, maxMtime)
+               }
        }
 
        for _, prefix := range []string{"f", "f15", "f15ac"} {
@@ -396,7 +415,7 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
 
        for _, prefix := range []string{"zero", "zip", "zilch"} {
                buf = new(bytes.Buffer)
-               v.IndexTo(prefix, buf)
+               err := v.IndexTo(prefix, buf)
                if err != nil {
                        t.Errorf("Got error on IndexTo with no such prefix %v", err.Error())
                } else if buf.Len() != 0 {
index 7aff85e59a4357acb1e27ce5386756feb96fa0e1..90189dc36cacab73276a322656755b32a580c909 100644 (file)
@@ -138,9 +138,8 @@ func (v *UnixVolume) Touch(loc string) error {
                return e
        }
        defer unlockfile(f)
-       now := time.Now().Unix()
-       utime := syscall.Utimbuf{now, now}
-       return syscall.Utime(p, &utime)
+       ts := syscall.NsecToTimespec(time.Now().UnixNano())
+       return syscall.UtimesNano(p, []syscall.Timespec{ts, ts})
 }
 
 // Mtime returns the stored timestamp for the given locator.
@@ -353,7 +352,7 @@ func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error {
                        _, err = fmt.Fprint(w,
                                name,
                                "+", fileInfo[0].Size(),
-                               " ", fileInfo[0].ModTime().Unix(),
+                               " ", fileInfo[0].ModTime().UnixNano(),
                                "\n")
                }
                blockdir.Close()