2 * Copyright (C) The Arvados Authors. All rights reserved.
4 * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
8 package org.arvados.client.logic.keep;
10 import com.google.common.collect.Lists;
11 import org.arvados.client.api.client.CollectionsApiClient;
12 import org.arvados.client.api.model.Collection;
13 import org.arvados.client.common.Characters;
14 import org.arvados.client.config.ConfigProvider;
15 import org.arvados.client.exception.ArvadosClientException;
16 import org.arvados.client.logic.collection.CollectionFactory;
17 import org.arvados.client.utils.FileMerge;
18 import org.arvados.client.utils.FileSplit;
19 import org.slf4j.Logger;
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.Objects;
25 import java.util.UUID;
27 import static java.util.stream.Collectors.toList;
29 public class FileUploader {
31 private final KeepClient keepClient;
32 private final CollectionsApiClient collectionsApiClient;
33 private final ConfigProvider config;
34 private final Logger log = org.slf4j.LoggerFactory.getLogger(FileUploader.class);
36 public FileUploader(KeepClient keepClient, CollectionsApiClient collectionsApiClient, ConfigProvider config) {
37 this.keepClient = keepClient;
38 this.collectionsApiClient = collectionsApiClient;
42 public Collection upload(List<File> sourceFiles, String collectionName, String projectUuid) {
43 List<String> locators = uploadToKeep(sourceFiles);
44 CollectionFactory collectionFactory = CollectionFactory.builder()
47 .projectUuid(projectUuid)
48 .manifestFiles(sourceFiles)
49 .manifestLocators(locators)
52 Collection newCollection = collectionFactory.create();
53 return collectionsApiClient.create(newCollection);
56 public Collection uploadToExistingCollection(List<File> files, String collectionUuid) {
57 List<String> locators = uploadToKeep(files);
58 Collection collectionBeforeUpload = collectionsApiClient.get(collectionUuid);
59 String oldManifest = collectionBeforeUpload.getManifestText();
61 CollectionFactory collectionFactory = CollectionFactory.builder()
64 .manifestLocators(locators).build();
66 String newPartOfManifestText = collectionFactory.create().getManifestText();
67 String newManifest = oldManifest + newPartOfManifestText;
69 collectionBeforeUpload.setManifestText(newManifest);
70 return collectionsApiClient.update(collectionBeforeUpload);
73 private List<String> uploadToKeep(List<File> files) {
74 File targetDir = config.getFileSplitDirectory();
75 File combinedFile = new File(targetDir.getAbsolutePath() + Characters.SLASH + UUID.randomUUID());
78 FileMerge.merge(files, combinedFile);
79 chunks = FileSplit.split(combinedFile, targetDir, config.getFileSplitSize());
80 } catch (IOException e) {
81 throw new ArvadosClientException("Cannot create file chunks for upload", e);
83 combinedFile.delete();
85 int copies = config.getNumberOfCopies();
86 int numRetries = config.getNumberOfRetries();
88 List<String> locators = Lists.newArrayList();
89 for (File chunk : chunks) {
91 locators.add(keepClient.put(chunk, copies, numRetries));
92 } catch (ArvadosClientException e) {
93 log.error("Problem occurred while uploading chunk file {}", chunk.getName(), e);
97 return locators.stream()
98 .filter(Objects::nonNull)