9397: get manifest segment for a subdir or file using manifest from sdk.

diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index f6656b4f2b8ae797e613110a27cdd938782ff941..18a6b858ea24d4a34a05dddc5375803622e59f63 100644 (file)
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -229,6 +229,49 @@ func parseManifestStream(s string) (m ManifestStream) {
        return
 }
 
+func (m *Manifest) ManifestStreamForPath(path string) string {
+       manifestForPath := ""
+       locators := []string{}
+       for ms := range m.StreamIter() {
+               if ms.StreamName != "./"+path {
+                       continue
+               }
+
+               locators = append(locators, ms.Blocks...)
+
+               currentName := ""
+               currentSpan := []uint64{0, 0}
+               for _, fss := range ms.FileStreamSegments {
+                       if fss.Name != currentName && currentName != "" {
+                               manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
+                       }
+
+                       if fss.Name != currentName {
+                               currentName = fss.Name
+                               currentSpan = []uint64{0, 0}
+                       }
+
+                       if currentSpan[1] == 0 {
+                               currentSpan = []uint64{fss.SegPos, fss.SegLen}
+                       } else {
+                               if currentSpan[1] == fss.SegPos {
+                                       currentSpan[1] += fss.SegLen
+                               } else {
+                                       manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]+fss.SegLen) + ":" + fss.Name + " "
+                                       currentSpan = []uint64{fss.SegPos, fss.SegPos + fss.SegLen}
+                               }
+                       }
+               }
+               manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
+       }
+
+       if len(locators) > 0 {
+               return "./" + path + " " + strings.Join(locators, " ") + " " + manifestForPath
+       }
+
+       return ""
+}
+
 func (m *Manifest) StreamIter() <-chan ManifestStream {
        ch := make(chan ManifestStream)
        go func(input string) {
@@ -272,10 +315,36 @@ func (m *Manifest) FileSegmentForPath(filepath string) string {
                dir = "./" + filepath[0:idx]
                file = filepath[idx+1:]
        }
+
+       var fileSegments []FileSegment
        for fs := range m.FileSegmentIterByName(filepath) {
-               return fmt.Sprintf("%v %v %v:%v:%v", dir, fs.Locator, fs.Offset, fs.Len, file)
+               fileSegments = append(fileSegments, *fs)
        }
-       return ""
+
+       if len(fileSegments) == 0 {
+               return ""
+       }
+
+       locators := ""
+       manifestForFile := ""
+       currentSpan := []int{0, 0}
+       for _, fs := range fileSegments {
+               locators += fs.Locator + " "
+
+               if currentSpan[1] == 0 {
+                       currentSpan = []int{fs.Offset, fs.Len}
+               } else {
+                       if currentSpan[1] == fs.Offset {
+                               currentSpan[1] += fs.Len
+                       } else {
+                               manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]+fs.Len) + ":" + file + " "
+                               currentSpan = []int{fs.Offset, fs.Offset + fs.Len}
+                       }
+               }
+       }
+       manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]) + ":" + file
+
+       return fmt.Sprintf("%v %v %v", dir, strings.Trim(locators, " "), manifestForFile)
 }
 
 // Blocks may appear multiple times within the same manifest if they

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 876df03d2c5ca83385e68684b76807d4ec622278..5a228e646ad4ac34539780852ee457bb94bf6b34 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -767,23 +767,19 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
                        pathSubdir = "." + mntPath[0:pathIdx]
                        pathFileName = mntPath[pathIdx+1:]
                }
-               streams := strings.Split(collection.ManifestText, "\n")
-               for _, stream := range streams {
-                       tokens := strings.Split(stream, " ")
-                       if tokens[0] == "."+mntPath {
-                               // path refers to this complete stream
-                               adjustedStream := strings.Replace(stream, "."+mntPath, "."+bindSuffix, -1)
-                               manifestText = adjustedStream + "\n"
-                               break
-                       } else {
-                               // look for a matching file in this stream
-                               fs := manifest.FileSegmentForPath(mntPath[1:])
-                               if fs != "" {
-                                       manifestText = strings.Replace(fs, ":"+pathFileName, ":"+bindFileName, -1)
-                                       manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
-                                       manifestText += "\n"
-                                       break
-                               }
+
+               manifestStreamForPath := manifest.ManifestStreamForPath(mntPath[1:])
+               if manifestStreamForPath != "" {
+                       // path refers to this complete stream
+                       adjustedStream := strings.Replace(manifestStreamForPath, "."+mntPath, "."+bindSuffix, -1)
+                       manifestText = adjustedStream + "\n"
+               } else {
+                       // look for a matching file in this stream
+                       fs := manifest.FileSegmentForPath(mntPath[1:])
+                       if fs != "" {
+                               manifestText = strings.Replace(fs, ":"+pathFileName, ":"+bindFileName, -1)
+                               manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
+                               manifestText += "\n"
                        }
                }
        }