sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileUploader.java

   1 /*
   2  * Copyright (C) The Arvados Authors. All rights reserved.
   3  *
   4  * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
   5  *
   6  */
   7
   8 package org.arvados.client.logic.keep;
   9
  10 import com.google.common.collect.Lists;
  11 import org.arvados.client.api.client.CollectionsApiClient;
  12 import org.arvados.client.api.model.Collection;
  13 import org.arvados.client.common.Characters;
  14 import org.arvados.client.config.ConfigProvider;
  15 import org.arvados.client.exception.ArvadosClientException;
  16 import org.arvados.client.logic.collection.CollectionFactory;
  17 import org.arvados.client.utils.FileMerge;
  18 import org.arvados.client.utils.FileSplit;
  19 import org.slf4j.Logger;
  20
  21 import java.io.File;
  22 import java.io.IOException;
  23 import java.util.List;
  24 import java.util.Objects;
  25 import java.util.UUID;
  26
  27 import static java.util.stream.Collectors.toList;
  28
  29 public class FileUploader {
  30
  31     private final KeepClient keepClient;
  32     private final CollectionsApiClient collectionsApiClient;
  33     private final ConfigProvider config;
  34     private final Logger log = org.slf4j.LoggerFactory.getLogger(FileUploader.class);
  35
  36     public FileUploader(KeepClient keepClient, CollectionsApiClient collectionsApiClient, ConfigProvider config) {
  37         this.keepClient = keepClient;
  38         this.collectionsApiClient = collectionsApiClient;
  39         this.config = config;
  40     }
  41
  42     public Collection upload(List<File> sourceFiles, String collectionName, String projectUuid) {
  43         List<String> locators = uploadToKeep(sourceFiles);
  44         CollectionFactory collectionFactory = CollectionFactory.builder()
  45                 .config(config)
  46                 .name(collectionName)
  47                 .projectUuid(projectUuid)
  48                 .manifestFiles(sourceFiles)
  49                 .manifestLocators(locators)
  50                 .build();
  51
  52         Collection newCollection = collectionFactory.create();
  53         return collectionsApiClient.create(newCollection);
  54     }
  55
  56     public Collection uploadToExistingCollection(List<File> files, String collectionUuid) {
  57         List<String> locators = uploadToKeep(files);
  58         Collection collectionBeforeUpload = collectionsApiClient.get(collectionUuid);
  59         String oldManifest = collectionBeforeUpload.getManifestText();
  60
  61         CollectionFactory collectionFactory = CollectionFactory.builder()
  62                 .config(config)
  63                 .manifestFiles(files)
  64                 .manifestLocators(locators).build();
  65
  66         String newPartOfManifestText = collectionFactory.create().getManifestText();
  67         String newManifest = oldManifest + newPartOfManifestText;
  68
  69         collectionBeforeUpload.setManifestText(newManifest);
  70         return collectionsApiClient.update(collectionBeforeUpload);
  71     }
  72
  73     private List<String> uploadToKeep(List<File> files) {
  74         File targetDir = config.getFileSplitDirectory();
  75         File combinedFile = new File(targetDir.getAbsolutePath() + Characters.SLASH + UUID.randomUUID());
  76         List<File> chunks;
  77         try {
  78             FileMerge.merge(files, combinedFile);
  79             chunks = FileSplit.split(combinedFile, targetDir, config.getFileSplitSize());
  80         } catch (IOException e) {
  81             throw new ArvadosClientException("Cannot create file chunks for upload", e);
  82         }
  83         combinedFile.delete();
  84
  85         int copies = config.getNumberOfCopies();
  86         int numRetries = config.getNumberOfRetries();
  87
  88         List<String> locators = Lists.newArrayList();
  89         for (File chunk : chunks) {
  90             try {
  91                 locators.add(keepClient.put(chunk, copies, numRetries));
  92             } catch (ArvadosClientException e) {
  93                 log.error("Problem occurred while uploading chunk file {}", chunk.getName(), e);
  94                 throw e;
  95             }
  96         }
  97         return locators.stream()
  98                 .filter(Objects::nonNull)
  99                 .collect(toList());
 100     }
 101 }