projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch 'master' into 9956-keepstore-config
[arvados.git]
/
sdk
/
go
/
manifest
/
manifest.go
diff --git
a/sdk/go/manifest/manifest.go
b/sdk/go/manifest/manifest.go
index d4b9830065e4345b1e717f2141e14bdf9d526e5d..22b1c974e634cd8229b645421ecb09480807c000 100644
(file)
--- a/
sdk/go/manifest/manifest.go
+++ b/
sdk/go/manifest/manifest.go
@@
-44,12
+44,19
@@
type FileSegment struct {
Len int
}
Len int
}
+// FileStreamSegment is a portion of a file described as a segment of a stream.
+type FileStreamSegment struct {
+ SegPos uint64
+ SegLen uint64
+ Name string
+}
+
// Represents a single line from a manifest.
type ManifestStream struct {
// Represents a single line from a manifest.
type ManifestStream struct {
- StreamName string
- Blocks []string
- File
Tokens []string
- Err error
+ StreamName
string
+ Blocks
[]string
+ File
StreamSegments []FileStreamSegment
+ Err
error
}
var escapeSeq = regexp.MustCompile(`\\([0-9]{3}|\\)`)
}
var escapeSeq = regexp.MustCompile(`\\([0-9]{3}|\\)`)
@@
-97,21
+104,21
@@
func ParseBlockLocator(s string) (b BlockLocator, err error) {
return
}
return
}
-func parseFile
Token(tok string) (segPos, segLen uint64, name string
, err error) {
+func parseFile
StreamSegment(tok string) (ft FileStreamSegment
, err error) {
parts := strings.SplitN(tok, ":", 3)
if len(parts) != 3 {
err = ErrInvalidToken
return
}
parts := strings.SplitN(tok, ":", 3)
if len(parts) != 3 {
err = ErrInvalidToken
return
}
-
s
egPos, err = strconv.ParseUint(parts[0], 10, 64)
+
ft.S
egPos, err = strconv.ParseUint(parts[0], 10, 64)
if err != nil {
return
}
if err != nil {
return
}
-
s
egLen, err = strconv.ParseUint(parts[1], 10, 64)
+
ft.S
egLen, err = strconv.ParseUint(parts[1], 10, 64)
if err != nil {
return
}
if err != nil {
return
}
-
n
ame = UnescapeName(parts[2])
+
ft.N
ame = UnescapeName(parts[2])
return
}
return
}
@@
-128,12
+135,11
@@
func (s *ManifestStream) sendFileSegmentIterByName(filepath string, ch chan<- *F
blockLens := make([]int, 0, len(s.Blocks))
// This is what streamName+"/"+fileName will look like:
target := "./" + filepath
blockLens := make([]int, 0, len(s.Blocks))
// This is what streamName+"/"+fileName will look like:
target := "./" + filepath
- for _, fTok := range s.FileTokens {
- wantPos, wantLen, name, err := parseFileToken(fTok)
- if err != nil {
- // Skip (!) invalid file tokens.
- continue
- }
+ for _, fTok := range s.FileStreamSegments {
+ wantPos := fTok.SegPos
+ wantLen := fTok.SegLen
+ name := fTok.Name
+
if s.StreamName+"/"+name != target {
continue
}
if s.StreamName+"/"+name != target {
continue
}
@@
-199,24
+205,25
@@
func parseManifestStream(s string) (m ManifestStream) {
}
}
m.Blocks = tokens[:i]
}
}
m.Blocks = tokens[:i]
-
m.FileTokens
= tokens[i:]
+
fileTokens :
= tokens[i:]
if len(m.Blocks) == 0 {
m.Err = fmt.Errorf("No block locators found")
return
}
if len(m.Blocks) == 0 {
m.Err = fmt.Errorf("No block locators found")
return
}
- if len(
m.F
ileTokens) == 0 {
+ if len(
f
ileTokens) == 0 {
m.Err = fmt.Errorf("No file tokens found")
return
}
m.Err = fmt.Errorf("No file tokens found")
return
}
- for _, ft := range
m.F
ileTokens {
-
_, _, _, err := parseFileToken
(ft)
+ for _, ft := range
f
ileTokens {
+
pft, err := parseFileStreamSegment
(ft)
if err != nil {
m.Err = fmt.Errorf("Invalid file token: %s", ft)
break
}
if err != nil {
m.Err = fmt.Errorf("Invalid file token: %s", ft)
break
}
+ m.FileStreamSegments = append(m.FileStreamSegments, pft)
}
return
}
return
@@
-258,7
+265,7
@@
func (m *Manifest) FileSegmentIterByName(filepath string) <-chan *FileSegment {
return ch
}
return ch
}
-// Blocks may appear mul
it
ple times within the same manifest if they
+// Blocks may appear mul
ti
ple times within the same manifest if they
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
//
// are used by multiple files. In that case this Iterator will output
// the same block multiple times.
//