Add 'sdk/java-v2/' from commit '55f103e336ca9fb8bf1720d2ef4ee8dd4e221118'

[arvados.git] / sdk / go / crunchrunner / upload.go

diff --git a/sdk/go/crunchrunner/upload.go b/sdk/go/crunchrunner/upload.go
index 06a6678e5c43bfc2d8f23d21a9dda05ed7af5b50..2848d1087c52684fe9ca5a79ec1f6f5c87b28b95 100644 (file)
--- a/sdk/go/crunchrunner/upload.go
+++ b/sdk/go/crunchrunner/upload.go
@@ -1,3 +1,7 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package main
 
 import (
@@ -5,14 +9,15 @@ import (
        "crypto/md5"
        "errors"
        "fmt"
-       "git.curoverse.com/arvados.git/sdk/go/keepclient"
-       "git.curoverse.com/arvados.git/sdk/go/manifest"
        "io"
        "log"
        "os"
        "path/filepath"
        "sort"
        "strings"
+
+       "git.curoverse.com/arvados.git/sdk/go/keepclient"
+       "git.curoverse.com/arvados.git/sdk/go/manifest"
 )
 
 type Block struct {
@@ -86,7 +91,26 @@ type ManifestWriter struct {
 }
 
 func (m *ManifestWriter) WalkFunc(path string, info os.FileInfo, err error) error {
-       if info.IsDir() {
+       if err != nil {
+               return err
+       }
+
+       targetPath, targetInfo := path, info
+       if info.Mode()&os.ModeSymlink != 0 {
+               // Update targetpath/info to reflect the symlink
+               // target, not the symlink itself
+               targetPath, err = filepath.EvalSymlinks(path)
+               if err != nil {
+                       return err
+               }
+               targetInfo, err = os.Stat(targetPath)
+               if err != nil {
+                       return fmt.Errorf("stat symlink %q target %q: %s", path, targetPath, err)
+               }
+       }
+
+       if targetInfo.Mode()&os.ModeType != 0 {
+               // Skip directories, pipes, other non-regular files
                return nil
        }
 
@@ -130,8 +154,8 @@ func (m *ManifestWriter) WalkFunc(path string, info os.FileInfo, err error) erro
 
        stream.offset += count
 
-       stream.ManifestStream.FileTokens = append(stream.ManifestStream.FileTokens,
-               fmt.Sprintf("%v:%v:%v", fileStart, count, fn))
+       stream.ManifestStream.FileStreamSegments = append(stream.ManifestStream.FileStreamSegments,
+               manifest.FileStreamSegment{uint64(fileStart), uint64(count), fn})
 
        return nil
 }
@@ -189,11 +213,11 @@ func (m *ManifestWriter) ManifestText() string {
                        buf.WriteString(" ")
                        buf.WriteString(b)
                }
-               for _, f := range v.FileTokens {
+               for _, f := range v.FileStreamSegments {
                        buf.WriteString(" ")
-                       f = strings.Replace(f, " ", "\\040", -1)
-                       f = strings.Replace(f, "\n", "", -1)
-                       buf.WriteString(f)
+                       name := strings.Replace(f.Name, " ", "\\040", -1)
+                       name = strings.Replace(name, "\n", "", -1)
+                       buf.WriteString(fmt.Sprintf("%d:%d:%s", f.SegPos, f.SegLen, name))
                }
                buf.WriteString("\n")
        }