18947: Refactor keep-web as arvados-server command.

[arvados.git] / sdk / go / arvados / collection.go

diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 35fd3fd7406fcd24b0f167d38fc9dd009f0e4d69..389fe4e4841b0006405aee66a697f607c414ce0d 100644 (file)
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -5,14 +5,21 @@
 package arvados
 
 import (
-       "bufio"
+       "bytes"
+       "crypto/md5"
        "fmt"
+       "regexp"
        "strings"
        "time"
 
        "git.arvados.org/arvados.git/sdk/go/blockdigest"
 )
 
+var (
+       UUIDMatch = regexp.MustCompile(`^[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}$`).MatchString
+       PDHMatch  = regexp.MustCompile(`^[0-9a-f]{32}\+\d+$`).MatchString
+)
+
 // Collection is an arvados#collection resource.
 type Collection struct {
        UUID                      string                 `json:"uuid"`
@@ -51,37 +58,43 @@ func (c Collection) resourceName() string {
 
 // SizedDigests returns the hash+size part of each data block
 // referenced by the collection.
+//
+// Zero-length blocks are not included.
 func (c *Collection) SizedDigests() ([]SizedDigest, error) {
-       manifestText := c.ManifestText
-       if manifestText == "" {
-               manifestText = c.UnsignedManifestText
+       manifestText := []byte(c.ManifestText)
+       if len(manifestText) == 0 {
+               manifestText = []byte(c.UnsignedManifestText)
        }
-       if manifestText == "" && c.PortableDataHash != "d41d8cd98f00b204e9800998ecf8427e+0" {
+       if len(manifestText) == 0 && c.PortableDataHash != "d41d8cd98f00b204e9800998ecf8427e+0" {
                // TODO: Check more subtle forms of corruption, too
                return nil, fmt.Errorf("manifest is missing")
        }
-       var sds []SizedDigest
-       scanner := bufio.NewScanner(strings.NewReader(manifestText))
-       scanner.Buffer(make([]byte, 1048576), len(manifestText))
-       for scanner.Scan() {
-               line := scanner.Text()
-               tokens := strings.Split(line, " ")
+       sds := make([]SizedDigest, 0, len(manifestText)/40)
+       for _, line := range bytes.Split(manifestText, []byte{'\n'}) {
+               if len(line) == 0 {
+                       continue
+               }
+               tokens := bytes.Split(line, []byte{' '})
                if len(tokens) < 3 {
                        return nil, fmt.Errorf("Invalid stream (<3 tokens): %q", line)
                }
                for _, token := range tokens[1:] {
-                       if !blockdigest.LocatorPattern.MatchString(token) {
+                       if !blockdigest.LocatorPattern.Match(token) {
                                // FIXME: ensure it's a file token
                                break
                        }
+                       if bytes.HasPrefix(token, []byte("d41d8cd98f00b204e9800998ecf8427e+0")) {
+                               // Exclude "empty block" placeholder
+                               continue
+                       }
                        // FIXME: shouldn't assume 32 char hash
-                       if i := strings.IndexRune(token[33:], '+'); i >= 0 {
+                       if i := bytes.IndexRune(token[33:], '+'); i >= 0 {
                                token = token[:33+i]
                        }
-                       sds = append(sds, SizedDigest(token))
+                       sds = append(sds, SizedDigest(string(token)))
                }
        }
-       return sds, scanner.Err()
+       return sds, nil
 }
 
 type CollectionList struct {
@@ -90,3 +103,50 @@ type CollectionList struct {
        Offset         int          `json:"offset"`
        Limit          int          `json:"limit"`
 }
+
+var (
+       blkRe = regexp.MustCompile(`^ [0-9a-f]{32}\+\d+`)
+       tokRe = regexp.MustCompile(` ?[^ ]*`)
+)
+
+// PortableDataHash computes the portable data hash of the given
+// manifest.
+func PortableDataHash(mt string) string {
+       h := md5.New()
+       size := 0
+       _ = tokRe.ReplaceAllFunc([]byte(mt), func(tok []byte) []byte {
+               if m := blkRe.Find(tok); m != nil {
+                       // write hash+size, ignore remaining block hints
+                       tok = m
+               }
+               n, err := h.Write(tok)
+               if err != nil {
+                       panic(err)
+               }
+               size += n
+               return nil
+       })
+       return fmt.Sprintf("%x+%d", h.Sum(nil), size)
+}
+
+// CollectionIDFromDNSName returns a UUID or PDH if s begins with a
+// UUID or URL-encoded PDH; otherwise "".
+func CollectionIDFromDNSName(s string) string {
+       // Strip domain.
+       if i := strings.IndexRune(s, '.'); i >= 0 {
+               s = s[:i]
+       }
+       // Names like {uuid}--collections.example.com serve the same
+       // purpose as {uuid}.collections.example.com but can reduce
+       // cost/effort of using [additional] wildcard certificates.
+       if i := strings.Index(s, "--"); i >= 0 {
+               s = s[:i]
+       }
+       if UUIDMatch(s) {
+               return s
+       }
+       if pdh := strings.Replace(s, "-", "+", 1); PDHMatch(pdh) {
+               return pdh
+       }
+       return ""
+}