+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
package main
// Originally based on sdk/go/crunchrunner/upload.go
"crypto/md5"
"errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "git.curoverse.com/arvados.git/sdk/go/manifest"
"io"
"log"
"os"
"path/filepath"
"strings"
"sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "git.curoverse.com/arvados.git/sdk/go/manifest"
)
// Block is a data block in a manifest stream
mtx sync.Mutex
}
-// WalkFunc walks a directory tree, uploads each file found and adds it to the
-// CollectionWriter.
-func (m *WalkUpload) WalkFunc(path string, info os.FileInfo, err error) error {
-
- if info.IsDir() {
- return nil
- }
-
+func (m *WalkUpload) UploadFile(path string, sourcePath string) error {
var dir string
- if len(path) > (len(m.stripPrefix) + len(info.Name()) + 1) {
- dir = path[len(m.stripPrefix)+1 : (len(path) - len(info.Name()) - 1)]
+ basename := filepath.Base(path)
+ if len(path) > (len(m.stripPrefix) + len(basename) + 1) {
+ dir = path[len(m.stripPrefix)+1 : (len(path) - len(basename) - 1)]
}
if dir == "" {
dir = "."
}
- fn := path[(len(path) - len(info.Name())):]
+ fn := path[(len(path) - len(basename)):]
+
+ info, err := os.Stat(sourcePath)
+ if err != nil {
+ return err
+ }
+ file, err := os.Open(sourcePath)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
if m.streamMap[dir] == nil {
m.streamMap[dir] = &CollectionFileWriter{
// Reset the CollectionFileWriter for a new file
fileWriter.NewFile(fn)
- file, err := os.Open(path)
- if err != nil {
- return err
- }
- defer file.Close()
-
m.status.Printf("Uploading %v/%v (%v bytes)", dir, fn, info.Size())
_, err = io.Copy(fileWriter, file)
if err != nil {
+ m.status.Printf("Uh oh")
return err
}
return nil
}
-func (cw *CollectionWriter) WriteTree(root string, status *log.Logger) (manifest string, err error) {
+func (cw *CollectionWriter) BeginUpload(root string, status *log.Logger) *WalkUpload {
streamMap := make(map[string]*CollectionFileWriter)
- wu := &WalkUpload{0, cw.IKeepClient, root, streamMap, status, nil, sync.Mutex{}}
- err = filepath.Walk(root, wu.WalkFunc)
-
- if err != nil {
- return "", err
- }
+ return &WalkUpload{0, cw.IKeepClient, root, streamMap, status, nil, sync.Mutex{}}
+}
+func (cw *CollectionWriter) EndUpload(wu *WalkUpload) {
cw.mtx.Lock()
- for _, st := range streamMap {
+ for _, st := range wu.streamMap {
cw.Streams = append(cw.Streams, st)
}
cw.mtx.Unlock()
-
- return cw.ManifestText()
}