projects
/
lightning.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Include numpy matrix filename in labels csv.
[lightning.git]
/
merge.go
diff --git
a/merge.go
b/merge.go
index 27b32308bf91cd69ae6e41aa2c45c9d4d1ea7ef9..968923711f3353fd0a37d36626a886446244ca93 100644
(file)
--- a/
merge.go
+++ b/
merge.go
@@
-67,12
+67,13
@@
func (cmd *merger) RunCommand(prog string, args []string, stdin io.Reader, stdou
return 1
}
runner := arvadosContainerRunner{
return 1
}
runner := arvadosContainerRunner{
- Name: "lightning
filter
",
+ Name: "lightning
merge
",
Client: arvados.NewClientFromEnv(),
ProjectUUID: *projectUUID,
Client: arvados.NewClientFromEnv(),
ProjectUUID: *projectUUID,
- RAM:
64
000000000,
+ RAM:
150
000000000,
VCPUs: 2,
Priority: *priority,
VCPUs: 2,
Priority: *priority,
+ APIAccess: true,
}
for i := range cmd.inputs {
err = runner.TranslatePaths(&cmd.inputs[i])
}
for i := range cmd.inputs {
err = runner.TranslatePaths(&cmd.inputs[i])
@@
-152,22
+153,21
@@
func (cmd *merger) doMerge() error {
var wg sync.WaitGroup
for _, input := range cmd.inputs {
var wg sync.WaitGroup
for _, input := range cmd.inputs {
- var infile io.ReadCloser
- if input == "-" {
- infile = ioutil.NopCloser(cmd.stdin)
- } else {
+ rdr := ioutil.NopCloser(cmd.stdin)
+ if input != "-" {
var err error
var err error
-
infile, err = os.O
pen(input)
+
rdr, err = o
pen(input)
if err != nil {
return err
}
if err != nil {
return err
}
- defer
infile
.Close()
+ defer
rdr
.Close()
}
}
+ rdr = ioutil.NopCloser(bufio.NewReaderSize(rdr, 8*1024*1024))
wg.Add(1)
go func(input string) {
defer wg.Done()
log.Printf("%s: reading", input)
wg.Add(1)
go func(input string) {
defer wg.Done()
log.Printf("%s: reading", input)
- err := cmd.tilelib.LoadGob(ctx,
infile
, strings.HasSuffix(input, ".gz"), nil)
+ err := cmd.tilelib.LoadGob(ctx,
rdr
, strings.HasSuffix(input, ".gz"), nil)
if err != nil {
cmd.setError(fmt.Errorf("%s: load failed: %w", input, err))
cancel()
if err != nil {
cmd.setError(fmt.Errorf("%s: load failed: %w", input, err))
cancel()