From: radhika <radhika@curoverse.com>
Date: Fri, 3 Feb 2017 21:00:33 +0000 (-0500)
Subject: 9397: get manifest segment for a subdir or file using manifest from sdk.
X-Git-Tag: 1.1.0~427^2~12
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/785c967e74a7dab0b29b276162d1d7513ce1cf6b?ds=sidebyside

9397: get manifest segment for a subdir or file using manifest from sdk.
---

diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index f6656b4f2b..18a6b858ea 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -229,6 +229,49 @@ func parseManifestStream(s string) (m ManifestStream) {
 	return
 }
 
+func (m *Manifest) ManifestStreamForPath(path string) string {
+	manifestForPath := ""
+	locators := []string{}
+	for ms := range m.StreamIter() {
+		if ms.StreamName != "./"+path {
+			continue
+		}
+
+		locators = append(locators, ms.Blocks...)
+
+		currentName := ""
+		currentSpan := []uint64{0, 0}
+		for _, fss := range ms.FileStreamSegments {
+			if fss.Name != currentName && currentName != "" {
+				manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
+			}
+
+			if fss.Name != currentName {
+				currentName = fss.Name
+				currentSpan = []uint64{0, 0}
+			}
+
+			if currentSpan[1] == 0 {
+				currentSpan = []uint64{fss.SegPos, fss.SegLen}
+			} else {
+				if currentSpan[1] == fss.SegPos {
+					currentSpan[1] += fss.SegLen
+				} else {
+					manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]+fss.SegLen) + ":" + fss.Name + " "
+					currentSpan = []uint64{fss.SegPos, fss.SegPos + fss.SegLen}
+				}
+			}
+		}
+		manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
+	}
+
+	if len(locators) > 0 {
+		return "./" + path + " " + strings.Join(locators, " ") + " " + manifestForPath
+	}
+
+	return ""
+}
+
 func (m *Manifest) StreamIter() <-chan ManifestStream {
 	ch := make(chan ManifestStream)
 	go func(input string) {
@@ -272,10 +315,36 @@ func (m *Manifest) FileSegmentForPath(filepath string) string {
 		dir = "./" + filepath[0:idx]
 		file = filepath[idx+1:]
 	}
+
+	var fileSegments []FileSegment
 	for fs := range m.FileSegmentIterByName(filepath) {
-		return fmt.Sprintf("%v %v %v:%v:%v", dir, fs.Locator, fs.Offset, fs.Len, file)
+		fileSegments = append(fileSegments, *fs)
 	}
-	return ""
+
+	if len(fileSegments) == 0 {
+		return ""
+	}
+
+	locators := ""
+	manifestForFile := ""
+	currentSpan := []int{0, 0}
+	for _, fs := range fileSegments {
+		locators += fs.Locator + " "
+
+		if currentSpan[1] == 0 {
+			currentSpan = []int{fs.Offset, fs.Len}
+		} else {
+			if currentSpan[1] == fs.Offset {
+				currentSpan[1] += fs.Len
+			} else {
+				manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]+fs.Len) + ":" + file + " "
+				currentSpan = []int{fs.Offset, fs.Offset + fs.Len}
+			}
+		}
+	}
+	manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]) + ":" + file
+
+	return fmt.Sprintf("%v %v %v", dir, strings.Trim(locators, " "), manifestForFile)
 }
 
 // Blocks may appear multiple times within the same manifest if they
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 876df03d2c..5a228e646a 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -767,23 +767,19 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
 			pathSubdir = "." + mntPath[0:pathIdx]
 			pathFileName = mntPath[pathIdx+1:]
 		}
-		streams := strings.Split(collection.ManifestText, "\n")
-		for _, stream := range streams {
-			tokens := strings.Split(stream, " ")
-			if tokens[0] == "."+mntPath {
-				// path refers to this complete stream
-				adjustedStream := strings.Replace(stream, "."+mntPath, "."+bindSuffix, -1)
-				manifestText = adjustedStream + "\n"
-				break
-			} else {
-				// look for a matching file in this stream
-				fs := manifest.FileSegmentForPath(mntPath[1:])
-				if fs != "" {
-					manifestText = strings.Replace(fs, ":"+pathFileName, ":"+bindFileName, -1)
-					manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
-					manifestText += "\n"
-					break
-				}
+
+		manifestStreamForPath := manifest.ManifestStreamForPath(mntPath[1:])
+		if manifestStreamForPath != "" {
+			// path refers to this complete stream
+			adjustedStream := strings.Replace(manifestStreamForPath, "."+mntPath, "."+bindSuffix, -1)
+			manifestText = adjustedStream + "\n"
+		} else {
+			// look for a matching file in this stream
+			fs := manifest.FileSegmentForPath(mntPath[1:])
+			if fs != "" {
+				manifestText = strings.Replace(fs, ":"+pathFileName, ":"+bindFileName, -1)
+				manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
+				manifestText += "\n"
 			}
 		}
 	}