tilelib.go

   1 // Copyright (C) The Lightning Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package lightning
   6
   7 import (
   8         "bufio"
   9         "bytes"
  10         "context"
  11         "encoding/gob"
  12         "fmt"
  13         "io"
  14         "os"
  15         "regexp"
  16         "runtime"
  17         "sort"
  18         "strings"
  19         "sync"
  20         "sync/atomic"
  21
  22         "github.com/klauspost/pgzip"
  23         log "github.com/sirupsen/logrus"
  24         "golang.org/x/crypto/blake2b"
  25 )
  26
  27 type tileVariantID uint16 // 1-based
  28
  29 type tileLibRef struct {
  30         Tag     tagID
  31         Variant tileVariantID
  32 }
  33
  34 type tileSeq map[string][]tileLibRef
  35
  36 func (tseq tileSeq) Variants() ([]tileVariantID, int, int) {
  37         maxtag := 0
  38         for _, refs := range tseq {
  39                 for _, ref := range refs {
  40                         if maxtag < int(ref.Tag) {
  41                                 maxtag = int(ref.Tag)
  42                         }
  43                 }
  44         }
  45         vars := make([]tileVariantID, maxtag+1)
  46         var kept, dropped int
  47         for _, refs := range tseq {
  48                 for _, ref := range refs {
  49                         if vars[int(ref.Tag)] != 0 {
  50                                 dropped++
  51                         } else {
  52                                 kept++
  53                         }
  54                         vars[int(ref.Tag)] = ref.Variant
  55                 }
  56         }
  57         return vars, kept, dropped
  58 }
  59
  60 type tileLibrary struct {
  61         retainNoCalls       bool
  62         skipOOO             bool
  63         retainTileSequences bool
  64
  65         taglib         *tagLibrary
  66         variant        [][][blake2b.Size256]byte
  67         refseqs        map[string]map[string][]tileLibRef
  68         compactGenomes map[string][]tileVariantID
  69         seq2           map[[2]byte]map[[blake2b.Size256]byte][]byte
  70         seq2lock       map[[2]byte]sync.Locker
  71         variants       int64
  72         // if non-nil, write out any tile variants added while tiling
  73         encoder *gob.Encoder
  74         // set Ref flag when writing new variants to encoder
  75         encodeRef bool
  76
  77         onAddTileVariant func(libref tileLibRef, hash [blake2b.Size256]byte, seq []byte) error
  78         onAddGenome      func(CompactGenome) error
  79         onAddRefseq      func(CompactSequence) error
  80
  81         mtx   sync.RWMutex
  82         vlock []sync.Locker
  83 }
  84
  85 func (tilelib *tileLibrary) loadTagSet(newtagset [][]byte) error {
  86         // Loading a tagset means either passing it through to the
  87         // output (if it's the first one we've seen), or just ensuring
  88         // it doesn't disagree with what we already have.
  89         if len(newtagset) == 0 {
  90                 return nil
  91         }
  92         tilelib.mtx.Lock()
  93         defer tilelib.mtx.Unlock()
  94         if tilelib.taglib == nil || tilelib.taglib.Len() == 0 {
  95                 tilelib.taglib = &tagLibrary{}
  96                 err := tilelib.taglib.setTags(newtagset)
  97                 if err != nil {
  98                         return err
  99                 }
 100                 if tilelib.encoder != nil {
 101                         err = tilelib.encoder.Encode(LibraryEntry{
 102                                 TagSet: newtagset,
 103                         })
 104                         if err != nil {
 105                                 return err
 106                         }
 107                 }
 108         } else if tilelib.taglib.Len() != len(newtagset) {
 109                 return fmt.Errorf("cannot merge libraries with differing tagsets")
 110         } else {
 111                 current := tilelib.taglib.Tags()
 112                 for i := range newtagset {
 113                         if !bytes.Equal(newtagset[i], current[i]) {
 114                                 return fmt.Errorf("cannot merge libraries with differing tagsets")
 115                         }
 116                 }
 117         }
 118         return nil
 119 }
 120
 121 func (tilelib *tileLibrary) loadTileVariants(tvs []TileVariant, variantmap map[tileLibRef]tileVariantID) error {
 122         for _, tv := range tvs {
 123                 // Assign a new variant ID (unique across all inputs)
 124                 // for each input variant.
 125                 variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence, tv.Ref).Variant
 126         }
 127         return nil
 128 }
 129
 130 func (tilelib *tileLibrary) loadCompactGenomes(cgs []CompactGenome, variantmap map[tileLibRef]tileVariantID) error {
 131         log.Debugf("loadCompactGenomes: %d", len(cgs))
 132         var wg sync.WaitGroup
 133         errs := make(chan error, 1)
 134         for _, cg := range cgs {
 135                 wg.Add(1)
 136                 cg := cg
 137                 go func() {
 138                         defer wg.Done()
 139                         for i, variant := range cg.Variants {
 140                                 if len(errs) > 0 {
 141                                         return
 142                                 }
 143                                 if variant == 0 {
 144                                         continue
 145                                 }
 146                                 tag := tagID(i / 2)
 147                                 newvariant, ok := variantmap[tileLibRef{Tag: tag, Variant: variant}]
 148                                 if !ok {
 149                                         err := fmt.Errorf("oops: genome %q has variant %d for tag %d, but that variant was not in its library", cg.Name, variant, tag)
 150                                         select {
 151                                         case errs <- err:
 152                                         default:
 153                                         }
 154                                         return
 155                                 }
 156                                 // log.Tracef("loadCompactGenomes: cg %s tag %d variant %d => %d", cg.Name, tag, variant, newvariant)
 157                                 cg.Variants[i] = newvariant
 158                         }
 159                         if tilelib.onAddGenome != nil {
 160                                 err := tilelib.onAddGenome(cg)
 161                                 if err != nil {
 162                                         select {
 163                                         case errs <- err:
 164                                         default:
 165                                         }
 166                                         return
 167                                 }
 168                         }
 169                         if tilelib.encoder != nil {
 170                                 err := tilelib.encoder.Encode(LibraryEntry{
 171                                         CompactGenomes: []CompactGenome{cg},
 172                                 })
 173                                 if err != nil {
 174                                         select {
 175                                         case errs <- err:
 176                                         default:
 177                                         }
 178                                         return
 179                                 }
 180                         }
 181                         if tilelib.compactGenomes != nil {
 182                                 tilelib.mtx.Lock()
 183                                 defer tilelib.mtx.Unlock()
 184                                 tilelib.compactGenomes[cg.Name] = cg.Variants
 185                         }
 186                 }()
 187         }
 188         wg.Wait()
 189         go close(errs)
 190         return <-errs
 191 }
 192
 193 func (tilelib *tileLibrary) loadCompactSequences(cseqs []CompactSequence, variantmap map[tileLibRef]tileVariantID) error {
 194         log.Infof("loadCompactSequences: %d todo", len(cseqs))
 195         for _, cseq := range cseqs {
 196                 log.Infof("loadCompactSequences: checking %s", cseq.Name)
 197                 for _, tseq := range cseq.TileSequences {
 198                         for i, libref := range tseq {
 199                                 if libref.Variant == 0 {
 200                                         // No variant (e.g., import
 201                                         // dropped tiles with
 202                                         // no-calls) = no translation.
 203                                         continue
 204                                 }
 205                                 v, ok := variantmap[libref]
 206                                 if !ok {
 207                                         return fmt.Errorf("oops: CompactSequence %q has variant %d for tag %d, but that variant was not in its library", cseq.Name, libref.Variant, libref.Tag)
 208                                 }
 209                                 tseq[i].Variant = v
 210                         }
 211                 }
 212                 if tilelib.encoder != nil {
 213                         if err := tilelib.encoder.Encode(LibraryEntry{
 214                                 CompactSequences: []CompactSequence{cseq},
 215                         }); err != nil {
 216                                 return err
 217                         }
 218                 }
 219                 if tilelib.onAddRefseq != nil {
 220                         err := tilelib.onAddRefseq(cseq)
 221                         if err != nil {
 222                                 return err
 223                         }
 224                 }
 225                 log.Infof("loadCompactSequences: checking %s done", cseq.Name)
 226         }
 227         tilelib.mtx.Lock()
 228         defer tilelib.mtx.Unlock()
 229         if tilelib.refseqs == nil {
 230                 tilelib.refseqs = map[string]map[string][]tileLibRef{}
 231         }
 232         for _, cseq := range cseqs {
 233                 tilelib.refseqs[cseq.Name] = cseq.TileSequences
 234         }
 235         log.Info("loadCompactSequences: done")
 236         return nil
 237 }
 238
 239 func allGobFiles(path string) ([]string, error) {
 240         var files []string
 241         f, err := open(path)
 242         if err != nil {
 243                 return nil, err
 244         }
 245         defer f.Close()
 246         fis, err := f.Readdir(-1)
 247         if err != nil {
 248                 return []string{path}, nil
 249         }
 250         for _, fi := range fis {
 251                 if fi.Name() == "." || fi.Name() == ".." {
 252                         continue
 253                 } else if child := path + "/" + fi.Name(); fi.IsDir() {
 254                         add, err := allGobFiles(child)
 255                         if err != nil {
 256                                 return nil, err
 257                         }
 258                         files = append(files, add...)
 259                 } else if strings.HasSuffix(child, ".gob") || strings.HasSuffix(child, ".gob.gz") {
 260                         files = append(files, child)
 261                 }
 262         }
 263         return files, nil
 264 }
 265
 266 func (tilelib *tileLibrary) LoadDir(ctx context.Context, path string) error {
 267         log.Infof("LoadDir: walk dir %s", path)
 268         files, err := allGobFiles(path)
 269         if err != nil {
 270                 return err
 271         }
 272         ctx, cancel := context.WithCancel(ctx)
 273         defer cancel()
 274         var mtx sync.Mutex
 275         allcgs := make([][]CompactGenome, len(files))
 276         allcseqs := make([][]CompactSequence, len(files))
 277         allvariantmap := map[tileLibRef]tileVariantID{}
 278         errs := make(chan error, len(files))
 279         log.Infof("LoadDir: read %d files", len(files))
 280         for fileno, path := range files {
 281                 fileno, path := fileno, path
 282                 go func() {
 283                         f, err := open(path)
 284                         if err != nil {
 285                                 errs <- err
 286                                 return
 287                         }
 288                         defer f.Close()
 289                         defer log.Infof("LoadDir: finished reading %s", path)
 290
 291                         var variantmap = map[tileLibRef]tileVariantID{}
 292                         var cgs []CompactGenome
 293                         var cseqs []CompactSequence
 294                         err = DecodeLibrary(f, strings.HasSuffix(path, ".gz"), func(ent *LibraryEntry) error {
 295                                 if ctx.Err() != nil {
 296                                         return ctx.Err()
 297                                 }
 298                                 if len(ent.TagSet) > 0 {
 299                                         mtx.Lock()
 300                                         if tilelib.taglib == nil || tilelib.taglib.Len() != len(ent.TagSet) {
 301                                                 // load first set of tags, or
 302                                                 // report mismatch if 2 sets
 303                                                 // have different #tags.
 304                                                 if err := tilelib.loadTagSet(ent.TagSet); err != nil {
 305                                                         mtx.Unlock()
 306                                                         return err
 307                                                 }
 308                                         }
 309                                         mtx.Unlock()
 310                                 }
 311                                 for _, tv := range ent.TileVariants {
 312                                         variantmap[tileLibRef{Tag: tv.Tag, Variant: tv.Variant}] = tilelib.getRef(tv.Tag, tv.Sequence, tv.Ref).Variant
 313                                 }
 314                                 cgs = append(cgs, ent.CompactGenomes...)
 315                                 cseqs = append(cseqs, ent.CompactSequences...)
 316                                 return nil
 317                         })
 318                         allcgs[fileno] = cgs
 319                         allcseqs[fileno] = cseqs
 320                         mtx.Lock()
 321                         defer mtx.Unlock()
 322                         for k, v := range variantmap {
 323                                 allvariantmap[k] = v
 324                         }
 325                         errs <- err
 326                 }()
 327         }
 328         for range files {
 329                 err := <-errs
 330                 if err != nil {
 331                         return err
 332                 }
 333         }
 334
 335         log.Info("LoadDir: loadCompactGenomes")
 336         var flatcgs []CompactGenome
 337         for _, cgs := range allcgs {
 338                 flatcgs = append(flatcgs, cgs...)
 339         }
 340         err = tilelib.loadCompactGenomes(flatcgs, allvariantmap)
 341         if err != nil {
 342                 return err
 343         }
 344
 345         log.Info("LoadDir: loadCompactSequences")
 346         var flatcseqs []CompactSequence
 347         for _, cseqs := range allcseqs {
 348                 flatcseqs = append(flatcseqs, cseqs...)
 349         }
 350         err = tilelib.loadCompactSequences(flatcseqs, allvariantmap)
 351         if err != nil {
 352                 return err
 353         }
 354
 355         log.Info("LoadDir done")
 356         return nil
 357 }
 358
 359 func (tilelib *tileLibrary) WriteDir(dir string) error {
 360         ntilefiles := 128
 361         nfiles := ntilefiles + len(tilelib.refseqs)
 362         files := make([]*os.File, nfiles)
 363         for i := range files {
 364                 f, err := os.OpenFile(fmt.Sprintf("%s/library.%04d.gob.gz", dir, i), os.O_CREATE|os.O_WRONLY, 0666)
 365                 if err != nil {
 366                         return err
 367                 }
 368                 defer f.Close()
 369                 files[i] = f
 370         }
 371         bufws := make([]*bufio.Writer, nfiles)
 372         for i := range bufws {
 373                 bufws[i] = bufio.NewWriterSize(files[i], 1<<26)
 374         }
 375         zws := make([]*pgzip.Writer, nfiles)
 376         for i := range zws {
 377                 zws[i] = pgzip.NewWriter(bufws[i])
 378                 defer zws[i].Close()
 379         }
 380         encoders := make([]*gob.Encoder, nfiles)
 381         for i := range encoders {
 382                 encoders[i] = gob.NewEncoder(zws[i])
 383         }
 384
 385         cgnames := make([]string, 0, len(tilelib.compactGenomes))
 386         for name := range tilelib.compactGenomes {
 387                 cgnames = append(cgnames, name)
 388         }
 389         sort.Strings(cgnames)
 390
 391         refnames := make([]string, 0, len(tilelib.refseqs))
 392         for name := range tilelib.refseqs {
 393                 refnames = append(refnames, name)
 394         }
 395         sort.Strings(refnames)
 396
 397         log.Infof("WriteDir: writing %d files", nfiles)
 398         ctx, cancel := context.WithCancel(context.Background())
 399         defer cancel()
 400         errs := make(chan error, nfiles)
 401         for start := range files {
 402                 start := start
 403                 go func() {
 404                         err := encoders[start].Encode(LibraryEntry{TagSet: tilelib.taglib.Tags()})
 405                         if err != nil {
 406                                 errs <- err
 407                                 return
 408                         }
 409                         if refidx := start - ntilefiles; refidx >= 0 {
 410                                 // write each ref to its own file
 411                                 // (they seem to load very slowly)
 412                                 name := refnames[refidx]
 413                                 errs <- encoders[start].Encode(LibraryEntry{CompactSequences: []CompactSequence{{
 414                                         Name:          name,
 415                                         TileSequences: tilelib.refseqs[name],
 416                                 }}})
 417                                 return
 418                         }
 419                         for i := start; i < len(cgnames); i += ntilefiles {
 420                                 err := encoders[start].Encode(LibraryEntry{CompactGenomes: []CompactGenome{{
 421                                         Name:     cgnames[i],
 422                                         Variants: tilelib.compactGenomes[cgnames[i]],
 423                                 }}})
 424                                 if err != nil {
 425                                         errs <- err
 426                                         return
 427                                 }
 428                         }
 429                         tvs := []TileVariant{}
 430                         for tag := start; tag < len(tilelib.variant) && ctx.Err() == nil; tag += ntilefiles {
 431                                 tvs = tvs[:0]
 432                                 for idx, hash := range tilelib.variant[tag] {
 433                                         tvs = append(tvs, TileVariant{
 434                                                 Tag:      tagID(tag),
 435                                                 Variant:  tileVariantID(idx + 1),
 436                                                 Blake2b:  hash,
 437                                                 Sequence: tilelib.hashSequence(hash),
 438                                         })
 439                                 }
 440                                 err := encoders[start].Encode(LibraryEntry{TileVariants: tvs})
 441                                 if err != nil {
 442                                         errs <- err
 443                                         return
 444                                 }
 445                         }
 446                         errs <- nil
 447                 }()
 448         }
 449         for range files {
 450                 err := <-errs
 451                 if err != nil {
 452                         return err
 453                 }
 454         }
 455         log.Info("WriteDir: flushing")
 456         for i := range zws {
 457                 err := zws[i].Close()
 458                 if err != nil {
 459                         return err
 460                 }
 461                 err = bufws[i].Flush()
 462                 if err != nil {
 463                         return err
 464                 }
 465                 err = files[i].Close()
 466                 if err != nil {
 467                         return err
 468                 }
 469         }
 470         log.Info("WriteDir: done")
 471         return nil
 472 }
 473
 474 // Load library data from rdr. Tile variants might be renumbered in
 475 // the process; in that case, genomes variants will be renumbered to
 476 // match.
 477 func (tilelib *tileLibrary) LoadGob(ctx context.Context, rdr io.Reader, gz bool) error {
 478         cgs := []CompactGenome{}
 479         cseqs := []CompactSequence{}
 480         variantmap := map[tileLibRef]tileVariantID{}
 481         err := DecodeLibrary(rdr, gz, func(ent *LibraryEntry) error {
 482                 if ctx.Err() != nil {
 483                         return ctx.Err()
 484                 }
 485                 if err := tilelib.loadTagSet(ent.TagSet); err != nil {
 486                         return err
 487                 }
 488                 if err := tilelib.loadTileVariants(ent.TileVariants, variantmap); err != nil {
 489                         return err
 490                 }
 491                 cgs = append(cgs, ent.CompactGenomes...)
 492                 cseqs = append(cseqs, ent.CompactSequences...)
 493                 return nil
 494         })
 495         if err != nil {
 496                 return err
 497         }
 498         if ctx.Err() != nil {
 499                 return ctx.Err()
 500         }
 501         err = tilelib.loadCompactGenomes(cgs, variantmap)
 502         if err != nil {
 503                 return err
 504         }
 505         err = tilelib.loadCompactSequences(cseqs, variantmap)
 506         if err != nil {
 507                 return err
 508         }
 509         return nil
 510 }
 511
 512 func (tilelib *tileLibrary) dump(out io.Writer) {
 513         printTV := func(tag int, variant tileVariantID) {
 514                 if variant < 1 {
 515                         fmt.Fprintf(out, " -")
 516                 } else if tag >= len(tilelib.variant) {
 517                         fmt.Fprintf(out, " (!tag=%d)", tag)
 518                 } else if int(variant) > len(tilelib.variant[tag]) {
 519                         fmt.Fprintf(out, " (tag=%d,!variant=%d)", tag, variant)
 520                 } else {
 521                         fmt.Fprintf(out, " %x", tilelib.variant[tag][variant-1][:8])
 522                 }
 523         }
 524         for refname, refseqs := range tilelib.refseqs {
 525                 for seqname, seq := range refseqs {
 526                         fmt.Fprintf(out, "ref %s %s", refname, seqname)
 527                         for _, libref := range seq {
 528                                 printTV(int(libref.Tag), libref.Variant)
 529                         }
 530                         fmt.Fprintf(out, "\n")
 531                 }
 532         }
 533         for name, cg := range tilelib.compactGenomes {
 534                 fmt.Fprintf(out, "cg %s", name)
 535                 for tag, variant := range cg {
 536                         printTV(tag/2, variant)
 537                 }
 538                 fmt.Fprintf(out, "\n")
 539         }
 540 }
 541
 542 type importStats struct {
 543         InputFile              string
 544         InputLabel             string
 545         InputLength            int
 546         InputCoverage          int
 547         PathLength             int
 548         DroppedOutOfOrderTiles int
 549 }
 550
 551 func (tilelib *tileLibrary) TileFasta(filelabel string, rdr io.Reader, matchChromosome *regexp.Regexp, isRef bool) (tileSeq, []importStats, error) {
 552         ret := tileSeq{}
 553         type jobT struct {
 554                 label string
 555                 fasta []byte
 556         }
 557         todo := make(chan jobT, 1)
 558         scanner := bufio.NewScanner(rdr)
 559         go func() {
 560                 defer close(todo)
 561                 var fasta []byte
 562                 var seqlabel string
 563                 for scanner.Scan() {
 564                         buf := scanner.Bytes()
 565                         if len(buf) > 0 && buf[0] == '>' {
 566                                 todo <- jobT{seqlabel, append([]byte(nil), fasta...)}
 567                                 seqlabel, fasta = strings.SplitN(string(buf[1:]), " ", 2)[0], fasta[:0]
 568                                 log.Debugf("%s %s reading fasta", filelabel, seqlabel)
 569                         } else {
 570                                 fasta = append(fasta, bytes.ToLower(buf)...)
 571                         }
 572                 }
 573                 todo <- jobT{seqlabel, fasta}
 574         }()
 575         type foundtag struct {
 576                 pos   int
 577                 tagid tagID
 578         }
 579         found := make([]foundtag, 2000000)
 580         path := make([]tileLibRef, 2000000)
 581         totalFoundTags := 0
 582         totalPathLen := 0
 583         skippedSequences := 0
 584         taglen := tilelib.taglib.TagLen()
 585         var stats []importStats
 586         for job := range todo {
 587                 if len(job.fasta) == 0 {
 588                         continue
 589                 } else if !matchChromosome.MatchString(job.label) {
 590                         skippedSequences++
 591                         continue
 592                 }
 593                 log.Debugf("%s %s tiling", filelabel, job.label)
 594
 595                 found = found[:0]
 596                 tilelib.taglib.FindAll(job.fasta, func(tagid tagID, pos, taglen int) {
 597                         found = append(found, foundtag{pos: pos, tagid: tagid})
 598                 })
 599                 totalFoundTags += len(found)
 600                 if len(found) == 0 {
 601                         log.Warnf("%s %s no tags found", filelabel, job.label)
 602                 }
 603
 604                 skipped := 0
 605                 if tilelib.skipOOO {
 606                         log.Infof("%s %s keeping longest increasing subsequence", filelabel, job.label)
 607                         keep := longestIncreasingSubsequence(len(found), func(i int) int { return int(found[i].tagid) })
 608                         for i, x := range keep {
 609                                 found[i] = found[x]
 610                         }
 611                         skipped = len(found) - len(keep)
 612                         found = found[:len(keep)]
 613                 }
 614
 615                 log.Infof("%s %s getting %d librefs", filelabel, job.label, len(found))
 616                 throttle := &throttle{Max: runtime.NumCPU()}
 617                 path = path[:len(found)]
 618                 var lowquality int64
 619                 for i, f := range found {
 620                         i, f := i, f
 621                         throttle.Acquire()
 622                         go func() {
 623                                 defer throttle.Release()
 624                                 var startpos, endpos int
 625                                 if i == 0 {
 626                                         startpos = 0
 627                                 } else {
 628                                         startpos = f.pos
 629                                 }
 630                                 if i == len(found)-1 {
 631                                         endpos = len(job.fasta)
 632                                 } else {
 633                                         endpos = found[i+1].pos + taglen
 634                                 }
 635                                 path[i] = tilelib.getRef(f.tagid, job.fasta[startpos:endpos], isRef)
 636                                 if countBases(job.fasta[startpos:endpos]) != endpos-startpos {
 637                                         atomic.AddInt64(&lowquality, 1)
 638                                 }
 639                         }()
 640                 }
 641                 throttle.Wait()
 642
 643                 log.Infof("%s %s copying path", filelabel, job.label)
 644
 645                 pathcopy := make([]tileLibRef, len(path))
 646                 copy(pathcopy, path)
 647                 ret[job.label] = pathcopy
 648
 649                 basesIn := countBases(job.fasta)
 650                 log.Infof("%s %s fasta in %d coverage in %d path len %d low-quality %d skipped-out-of-order %d", filelabel, job.label, len(job.fasta), basesIn, len(path), lowquality, skipped)
 651                 stats = append(stats, importStats{
 652                         InputFile:              filelabel,
 653                         InputLabel:             job.label,
 654                         InputLength:            len(job.fasta),
 655                         InputCoverage:          basesIn,
 656                         PathLength:             len(path),
 657                         DroppedOutOfOrderTiles: skipped,
 658                 })
 659
 660                 totalPathLen += len(path)
 661         }
 662         log.Printf("%s tiled with total path len %d in %d sequences (skipped %d sequences that did not match chromosome regexp, skipped %d out-of-order tags)", filelabel, totalPathLen, len(ret), skippedSequences, totalFoundTags-totalPathLen)
 663         return ret, stats, scanner.Err()
 664 }
 665
 666 func (tilelib *tileLibrary) Len() int64 {
 667         return atomic.LoadInt64(&tilelib.variants)
 668 }
 669
 670 // Return a tileLibRef for a tile with the given tag and sequence,
 671 // adding the sequence to the library if needed.
 672 func (tilelib *tileLibrary) getRef(tag tagID, seq []byte, usedByRef bool) tileLibRef {
 673         dropSeq := false
 674         if !tilelib.retainNoCalls {
 675                 for _, b := range seq {
 676                         if b != 'a' && b != 'c' && b != 'g' && b != 't' {
 677                                 dropSeq = true
 678                                 break
 679                         }
 680                 }
 681         }
 682         seqhash := blake2b.Sum256(seq)
 683         var vlock sync.Locker
 684
 685         tilelib.mtx.RLock()
 686         if len(tilelib.vlock) > int(tag) {
 687                 vlock = tilelib.vlock[tag]
 688         }
 689         tilelib.mtx.RUnlock()
 690
 691         if vlock != nil {
 692                 vlock.Lock()
 693                 for i, varhash := range tilelib.variant[tag] {
 694                         if varhash == seqhash {
 695                                 vlock.Unlock()
 696                                 return tileLibRef{Tag: tag, Variant: tileVariantID(i + 1)}
 697                         }
 698                 }
 699                 vlock.Unlock()
 700         } else {
 701                 tilelib.mtx.Lock()
 702                 if tilelib.variant == nil && tilelib.taglib != nil {
 703                         tilelib.variant = make([][][blake2b.Size256]byte, tilelib.taglib.Len())
 704                         tilelib.vlock = make([]sync.Locker, tilelib.taglib.Len())
 705                         for i := range tilelib.vlock {
 706                                 tilelib.vlock[i] = new(sync.Mutex)
 707                         }
 708                 }
 709                 if int(tag) >= len(tilelib.variant) {
 710                         oldlen := len(tilelib.vlock)
 711                         for i := 0; i < oldlen; i++ {
 712                                 tilelib.vlock[i].Lock()
 713                         }
 714                         // If we haven't seen the tag library yet (as
 715                         // in a merge), tilelib.taglib.Len() is
 716                         // zero. We can still behave correctly, we
 717                         // just need to expand the tilelib.variant and
 718                         // tilelib.vlock slices as needed.
 719                         if int(tag) >= cap(tilelib.variant) {
 720                                 // Allocate 2x capacity.
 721                                 newslice := make([][][blake2b.Size256]byte, int(tag)+1, (int(tag)+1)*2)
 722                                 copy(newslice, tilelib.variant)
 723                                 tilelib.variant = newslice[:int(tag)+1]
 724                                 newvlock := make([]sync.Locker, int(tag)+1, (int(tag)+1)*2)
 725                                 copy(newvlock, tilelib.vlock)
 726                                 tilelib.vlock = newvlock[:int(tag)+1]
 727                         } else {
 728                                 // Use previously allocated capacity,
 729                                 // avoiding copy.
 730                                 tilelib.variant = tilelib.variant[:int(tag)+1]
 731                                 tilelib.vlock = tilelib.vlock[:int(tag)+1]
 732                         }
 733                         for i := oldlen; i < len(tilelib.vlock); i++ {
 734                                 tilelib.vlock[i] = new(sync.Mutex)
 735                         }
 736                         for i := 0; i < oldlen; i++ {
 737                                 tilelib.vlock[i].Unlock()
 738                         }
 739                 }
 740                 vlock = tilelib.vlock[tag]
 741                 tilelib.mtx.Unlock()
 742         }
 743
 744         vlock.Lock()
 745         for i, varhash := range tilelib.variant[tag] {
 746                 if varhash == seqhash {
 747                         vlock.Unlock()
 748                         return tileLibRef{Tag: tag, Variant: tileVariantID(i + 1)}
 749                 }
 750         }
 751         atomic.AddInt64(&tilelib.variants, 1)
 752         tilelib.variant[tag] = append(tilelib.variant[tag], seqhash)
 753         variant := tileVariantID(len(tilelib.variant[tag]))
 754         vlock.Unlock()
 755
 756         if tilelib.retainTileSequences && !dropSeq {
 757                 seqCopy := append([]byte(nil), seq...)
 758                 if tilelib.seq2 == nil {
 759                         tilelib.mtx.Lock()
 760                         if tilelib.seq2 == nil {
 761                                 tilelib.seq2lock = map[[2]byte]sync.Locker{}
 762                                 m := map[[2]byte]map[[blake2b.Size256]byte][]byte{}
 763                                 var k [2]byte
 764                                 for i := 0; i < 256; i++ {
 765                                         k[0] = byte(i)
 766                                         for j := 0; j < 256; j++ {
 767                                                 k[1] = byte(j)
 768                                                 m[k] = map[[blake2b.Size256]byte][]byte{}
 769                                                 tilelib.seq2lock[k] = &sync.Mutex{}
 770                                         }
 771                                 }
 772                                 tilelib.seq2 = m
 773                         }
 774                         tilelib.mtx.Unlock()
 775                 }
 776                 var k [2]byte
 777                 copy(k[:], seqhash[:])
 778                 locker := tilelib.seq2lock[k]
 779                 locker.Lock()
 780                 tilelib.seq2[k][seqhash] = seqCopy
 781                 locker.Unlock()
 782         }
 783
 784         saveSeq := seq
 785         if dropSeq {
 786                 // Save the hash, but not the sequence
 787                 saveSeq = nil
 788         }
 789         if tilelib.encoder != nil {
 790                 tilelib.encoder.Encode(LibraryEntry{
 791                         TileVariants: []TileVariant{{
 792                                 Tag:      tag,
 793                                 Ref:      usedByRef,
 794                                 Variant:  variant,
 795                                 Blake2b:  seqhash,
 796                                 Sequence: saveSeq,
 797                         }},
 798                 })
 799         }
 800         if tilelib.onAddTileVariant != nil {
 801                 tilelib.onAddTileVariant(tileLibRef{tag, variant}, seqhash, saveSeq)
 802         }
 803         return tileLibRef{Tag: tag, Variant: variant}
 804 }
 805
 806 func (tilelib *tileLibrary) hashSequence(hash [blake2b.Size256]byte) []byte {
 807         var partition [2]byte
 808         copy(partition[:], hash[:])
 809         return tilelib.seq2[partition][hash]
 810 }
 811
 812 func (tilelib *tileLibrary) TileVariantSequence(libref tileLibRef) []byte {
 813         if libref.Variant == 0 || len(tilelib.variant) <= int(libref.Tag) || len(tilelib.variant[libref.Tag]) < int(libref.Variant) {
 814                 return nil
 815         }
 816         return tilelib.hashSequence(tilelib.variant[libref.Tag][libref.Variant-1])
 817 }
 818
 819 // Tidy deletes unreferenced tile variants and renumbers variants so
 820 // more common variants have smaller IDs.
 821 func (tilelib *tileLibrary) Tidy() {
 822         log.Print("Tidy: compute inref")
 823         inref := map[tileLibRef]bool{}
 824         for _, refseq := range tilelib.refseqs {
 825                 for _, librefs := range refseq {
 826                         for _, libref := range librefs {
 827                                 inref[libref] = true
 828                         }
 829                 }
 830         }
 831         log.Print("Tidy: compute remap")
 832         remap := make([][]tileVariantID, len(tilelib.variant))
 833         throttle := throttle{Max: runtime.NumCPU() + 1}
 834         for tag, oldvariants := range tilelib.variant {
 835                 tag, oldvariants := tagID(tag), oldvariants
 836                 if tag%1000000 == 0 {
 837                         log.Printf("Tidy: tag %d", tag)
 838                 }
 839                 throttle.Acquire()
 840                 go func() {
 841                         defer throttle.Release()
 842                         uses := make([]int, len(oldvariants))
 843                         for _, cg := range tilelib.compactGenomes {
 844                                 for phase := 0; phase < 2; phase++ {
 845                                         cgi := int(tag)*2 + phase
 846                                         if cgi < len(cg) && cg[cgi] > 0 {
 847                                                 uses[cg[cgi]-1]++
 848                                         }
 849                                 }
 850                         }
 851
 852                         // Compute desired order of variants:
 853                         // neworder[x] == index in oldvariants that
 854                         // should move to position x.
 855                         neworder := make([]int, len(oldvariants))
 856                         for i := range neworder {
 857                                 neworder[i] = i
 858                         }
 859                         sort.Slice(neworder, func(i, j int) bool {
 860                                 if cmp := uses[neworder[i]] - uses[neworder[j]]; cmp != 0 {
 861                                         return cmp > 0
 862                                 } else {
 863                                         return bytes.Compare(oldvariants[neworder[i]][:], oldvariants[neworder[j]][:]) < 0
 864                                 }
 865                         })
 866
 867                         // Replace tilelib.variant[tag] with a new
 868                         // re-ordered slice of hashes, and make a
 869                         // mapping from old to new variant IDs.
 870                         remaptag := make([]tileVariantID, len(oldvariants)+1)
 871                         newvariants := make([][blake2b.Size256]byte, 0, len(neworder))
 872                         for _, oldi := range neworder {
 873                                 if uses[oldi] > 0 || inref[tileLibRef{Tag: tag, Variant: tileVariantID(oldi + 1)}] {
 874                                         newvariants = append(newvariants, oldvariants[oldi])
 875                                         remaptag[oldi+1] = tileVariantID(len(newvariants))
 876                                 }
 877                         }
 878                         tilelib.variant[tag] = newvariants
 879                         remap[tag] = remaptag
 880                 }()
 881         }
 882         throttle.Wait()
 883
 884         // Apply remap to genomes and reference sequences, so they
 885         // refer to the same tile variants using the changed IDs.
 886         log.Print("Tidy: apply remap")
 887         var wg sync.WaitGroup
 888         for _, cg := range tilelib.compactGenomes {
 889                 cg := cg
 890                 wg.Add(1)
 891                 go func() {
 892                         defer wg.Done()
 893                         for idx, variant := range cg {
 894                                 cg[idx] = remap[tagID(idx/2)][variant]
 895                         }
 896                 }()
 897         }
 898         for _, refcs := range tilelib.refseqs {
 899                 for _, refseq := range refcs {
 900                         refseq := refseq
 901                         wg.Add(1)
 902                         go func() {
 903                                 defer wg.Done()
 904                                 for i, tv := range refseq {
 905                                         refseq[i].Variant = remap[tv.Tag][tv.Variant]
 906                                 }
 907                         }()
 908                 }
 909         }
 910         wg.Wait()
 911         log.Print("Tidy: done")
 912 }
 913
 914 func countBases(seq []byte) int {
 915         n := 0
 916         for _, c := range seq {
 917                 if isbase[c] {
 918                         n++
 919                 }
 920         }
 921         return n
 922 }