// copied from the local filesystem.
//
// Symlinks to mounted collections, and any collections mounted under
-// ctrOutputDir, are copied by transforming the relevant parts of the
-// existing manifests, without moving any data around.
+// ctrOutputDir, are copied by reference, without moving any data
+// around.
//
// Symlinks to other parts of the container's filesystem result in
// errors.
secretMounts map[string]arvados.Mount
logger printfer
- dirs []string
- files []filetodo
- manifest string
+ dirs []string
+ files []filetodo
+ staged arvados.CollectionFileSystem
manifestCache map[string]string
}
// Copy copies data as needed, and returns a new manifest.
+//
+// Copy should not be called more than once.
func (cp *copier) Copy() (string, error) {
- err := cp.walkMount("", cp.ctrOutputDir, limitFollowSymlinks, true)
+ var err error
+ cp.staged, err = (&arvados.Collection{}).FileSystem(cp.client, cp.keepClient)
if err != nil {
- return "", fmt.Errorf("error scanning files to copy to output: %v", err)
+ return "", fmt.Errorf("error creating Collection.FileSystem: %v", err)
}
- collfs, err := (&arvados.Collection{ManifestText: cp.manifest}).FileSystem(cp.client, cp.keepClient)
+ err = cp.walkMount("", cp.ctrOutputDir, limitFollowSymlinks, true)
if err != nil {
- return "", fmt.Errorf("error creating Collection.FileSystem: %v", err)
+ return "", fmt.Errorf("error scanning files to copy to output: %v", err)
}
- // Remove files/dirs that don't match globs (the ones that
- // were added during cp.walkMount() by copying subtree
- // manifests into cp.manifest).
- err = cp.applyGlobsToCollectionFS(collfs)
+ // Remove files/dirs that don't match globs (the files/dirs
+ // that were added during cp.walkMount() by copying subtree
+ // manifests into cp.staged).
+ err = cp.applyGlobsToStaged()
if err != nil {
return "", fmt.Errorf("error while removing non-matching files from output collection: %w", err)
}
- // Remove files/dirs that don't match globs (the ones that are
- // stored on the local filesystem and would need to be copied
- // in copyFile() below).
+ // Remove files/dirs that don't match globs (the files/dirs
+ // that are stored on the local filesystem and would need to
+ // be copied in copyFile() below).
cp.applyGlobsToFilesAndDirs()
for _, d := range cp.dirs {
- err = collfs.Mkdir(d, 0777)
+ err = cp.staged.Mkdir(d, 0777)
if err != nil && err != os.ErrExist {
return "", fmt.Errorf("error making directory %q in output collection: %v", d, err)
}
// open so f's data can be packed with it).
dir, _ := filepath.Split(f.dst)
if dir != lastparentdir || unflushed > keepclient.BLOCKSIZE {
- if err := collfs.Flush("/"+lastparentdir, dir != lastparentdir); err != nil {
+ if err := cp.staged.Flush("/"+lastparentdir, dir != lastparentdir); err != nil {
return "", fmt.Errorf("error flushing output collection file data: %v", err)
}
unflushed = 0
}
lastparentdir = dir
- n, err := cp.copyFile(collfs, f)
+ n, err := cp.copyFile(cp.staged, f)
if err != nil {
return "", fmt.Errorf("error copying file %q into output collection: %v", f, err)
}
unflushed += n
}
- return collfs.MarshalManifest(".")
+ return cp.staged.MarshalManifest(".")
}
func (cp *copier) matchGlobs(path string, isDir bool) bool {
cp.files = keepfiles
}
-// Delete files in collfs that do not match cp.globs. Also delete
+// Delete files in cp.staged that do not match cp.globs. Also delete
// directories that are empty (after deleting non-matching files) and
// do not match cp.globs themselves.
-func (cp *copier) applyGlobsToCollectionFS(collfs arvados.CollectionFileSystem) error {
+func (cp *copier) applyGlobsToStaged() error {
if len(cp.globs) == 0 {
return nil
}
include := make(map[string]bool)
- err := fs.WalkDir(arvados.FS(collfs), "", func(path string, ent fs.DirEntry, err error) error {
+ err := fs.WalkDir(arvados.FS(cp.staged), "", func(path string, ent fs.DirEntry, err error) error {
if cp.matchGlobs(path, ent.IsDir()) {
for i, c := range path {
if i > 0 && c == '/' {
if err != nil {
return err
}
- err = fs.WalkDir(arvados.FS(collfs), "", func(path string, ent fs.DirEntry, err error) error {
+ err = fs.WalkDir(arvados.FS(cp.staged), "", func(path string, ent fs.DirEntry, err error) error {
if err != nil || path == "" {
return err
}
if !include[path] {
- err := collfs.RemoveAll(path)
+ err := cp.staged.RemoveAll(path)
if err != nil {
return err
}
return n, dst.Close()
}
-// Append to cp.manifest, cp.files, and cp.dirs so as to copy src (an
+// Add to cp.staged, cp.files, and cp.dirs so as to copy src (an
// absolute path in the container's filesystem) to dest (an absolute
// path in the output collection, or "" for output root).
//
if err != nil {
return err
}
- tmpfs, err = (&arvados.Collection{}).FileSystem(cp.client, cp.keepClient)
- if err != nil {
- return err
- }
// Create ancestors of dest, if necessary.
for i, c := range dest {
if i > 0 && c == '/' {
- err = tmpfs.Mkdir(dest[:i], 0777)
+ err = cp.staged.Mkdir(dest[:i], 0777)
if err != nil && !os.IsExist(err) {
return err
}
}
}
- err = arvados.Splice(tmpfs, dest, snap)
- if err != nil {
- return err
- }
- mtxt, err := tmpfs.MarshalManifest(".")
- if err != nil {
- return err
- }
- cp.manifest += mtxt
- return nil
+ return arvados.Splice(cp.staged, dest, snap)
}
func (cp *copier) walkMountsBelow(dest, src string) error {
projects / arvados.git / blobdiff