Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>
"errors"
"fmt"
"io/ioutil"
"errors"
"fmt"
"io/ioutil"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/keepclient"
+ log "github.com/sirupsen/logrus"
"golang.org/x/crypto/blake2b"
"golang.org/x/net/websocket"
)
"golang.org/x/crypto/blake2b"
"golang.org/x/net/websocket"
)
"net/http"
_ "net/http/pprof"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/kshedden/gonpy"
"net/http"
_ "net/http/pprof"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
"github.com/kshedden/gonpy"
+ log "github.com/sirupsen/logrus"
)
type exportNumpy struct{}
)
type exportNumpy struct{}
"net/http"
_ "net/http/pprof"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
"net/http"
_ "net/http/pprof"
"os"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ log "github.com/sirupsen/logrus"
github.com/kshedden/gonpy v0.0.0-20190510000443-66c21fac4672
github.com/prometheus/client_golang v1.5.0 // indirect
github.com/prometheus/procfs v0.0.10 // indirect
github.com/kshedden/gonpy v0.0.0-20190510000443-66c21fac4672
github.com/prometheus/client_golang v1.5.0 // indirect
github.com/prometheus/procfs v0.0.10 // indirect
+ github.com/sirupsen/logrus v1.4.2
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
"net/http"
_ "net/http/pprof"
"os"
"net/http"
_ "net/http/pprof"
"os"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ log "github.com/sirupsen/logrus"
- for range time.Tick(10 * time.Second) {
+ for range time.Tick(10 * time.Minute) {
log.Printf("tilelib.Len() == %d", tilelib.Len())
}
}()
log.Printf("tilelib.Len() == %d", tilelib.Len())
}
}()
+ log "github.com/sirupsen/logrus"
"golang.org/x/crypto/blake2b"
)
"golang.org/x/crypto/blake2b"
)
if len(buf) == 0 || buf[0] == '>' {
todo <- jobT{seqlabel, fasta}
seqlabel, fasta = string(buf[1:]), nil
if len(buf) == 0 || buf[0] == '>' {
todo <- jobT{seqlabel, fasta}
seqlabel, fasta = string(buf[1:]), nil
- log.Printf("%s %s reading fasta", filelabel, seqlabel)
+ log.Debugf("%s %s reading fasta", filelabel, seqlabel)
} else {
fasta = append(fasta, bytes.ToLower(buf)...)
}
} else {
fasta = append(fasta, bytes.ToLower(buf)...)
}
todo <- jobT{seqlabel, fasta}
}()
path := make([]tileLibRef, 2000000)
todo <- jobT{seqlabel, fasta}
}()
path := make([]tileLibRef, 2000000)
+ totalPathLen := 0
+ skippedSequences := 0
- if len(job.fasta) == 0 || strings.Contains(job.label, "_") {
+ if len(job.fasta) == 0 {
+ continue
+ } else if strings.Contains(job.label, "_") {
+ skippedSequences++
- log.Printf("%s %s tiling", filelabel, job.label)
+ log.Debugf("%s %s tiling", filelabel, job.label)
path = path[:0]
tilestart := -1 // position in fasta of tile that ends here
tiletagid := tagID(-1) // tag id starting tile that ends here
path = path[:0]
tilestart := -1 // position in fasta of tile that ends here
tiletagid := tagID(-1) // tag id starting tile that ends here
pathcopy := make([]tileLibRef, len(path))
copy(pathcopy, path)
ret[job.label] = pathcopy
pathcopy := make([]tileLibRef, len(path))
copy(pathcopy, path)
ret[job.label] = pathcopy
- log.Printf("%s %s tiled with path len %d", filelabel, job.label, len(path))
+ log.Debugf("%s %s tiled with path len %d", filelabel, job.label, len(path))
+ totalPathLen += len(path)
+ log.Printf("%s tiled with total path len %d in %d sequences (skipped %d sequences with '_' in name)", filelabel, totalPathLen, len(ret), skippedSequences)
return ret, scanner.Err()
}
return ret, scanner.Err()
}