2 * Copyright (C) The Arvados Authors. All rights reserved.
4 * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
8 package org.arvados.client.logic.keep;
10 import com.fasterxml.jackson.databind.ObjectMapper;
11 import org.arvados.client.api.client.CollectionsApiClient;
12 import org.arvados.client.api.client.KeepWebApiClient;
13 import org.arvados.client.api.model.Collection;
14 import org.arvados.client.common.Characters;
15 import org.arvados.client.logic.collection.FileToken;
16 import org.arvados.client.logic.collection.ManifestDecoder;
17 import org.arvados.client.logic.collection.ManifestStream;
18 import org.arvados.client.test.utils.FileTestUtils;
19 import org.arvados.client.utils.FileMerge;
20 import org.apache.commons.io.FileUtils;
21 import org.junit.After;
22 import org.junit.Assert;
23 import org.junit.Before;
24 import org.junit.Test;
25 import org.junit.runner.RunWith;
26 import org.mockito.InjectMocks;
27 import org.mockito.Mock;
28 import org.mockito.junit.MockitoJUnitRunner;
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.List;
35 import java.util.UUID;
37 import static org.arvados.client.test.utils.FileTestUtils.*;
38 import static org.assertj.core.api.Assertions.assertThat;
39 import static org.mockito.Mockito.when;
41 @RunWith(MockitoJUnitRunner.class)
42 public class FileDownloaderTest {
44 static final ObjectMapper MAPPER = new ObjectMapper().findAndRegisterModules();
45 private Collection collectionToDownload;
46 private ManifestStream manifestStream;
49 private CollectionsApiClient collectionsApiClient;
51 private KeepClient keepClient;
53 private KeepWebApiClient keepWebApiClient;
55 private ManifestDecoder manifestDecoder;
57 private FileDownloader fileDownloader;
60 public void setUp() throws Exception {
61 FileTestUtils.createDirectory(FILE_SPLIT_TEST_DIR);
62 FileTestUtils.createDirectory(FILE_DOWNLOAD_TEST_DIR);
64 collectionToDownload = prepareCollection();
65 manifestStream = prepareManifestStream();
69 public void downloadingAllFilesFromCollectionWorksProperly() throws Exception {
71 List<File> files = generatePredefinedFiles();
72 byte[] dataChunk = prepareDataChunk(files);
75 when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
76 when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
77 when(keepClient.getDataChunk(manifestStream.getKeepLocators().get(0))).thenReturn(dataChunk);
80 List<File> downloadedFiles = fileDownloader.downloadFilesFromCollection(collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
83 Assert.assertEquals(3, downloadedFiles.size()); // 3 files downloaded
85 File collectionDir = new File(FILE_DOWNLOAD_TEST_DIR + Characters.SLASH + collectionToDownload.getUuid());
86 Assert.assertTrue(collectionDir.exists()); // collection directory created
88 // 3 files correctly saved
89 assertThat(downloadedFiles).allMatch(File::exists);
91 for(int i = 0; i < downloadedFiles.size(); i ++) {
92 File downloaded = new File(collectionDir + Characters.SLASH + files.get(i).getName());
93 Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloaded), FileUtils.readFileToByteArray(files.get(i)));
98 public void downloadingSingleFileFromKeepWebWorksCorrectly() throws Exception{
100 File file = generatePredefinedFiles().get(0);
103 when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
104 when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
105 when(keepWebApiClient.download(collectionToDownload.getUuid(), file.getName())).thenReturn(FileUtils.readFileToByteArray(file));
108 File downloadedFile = fileDownloader.downloadSingleFileUsingKeepWeb(file.getName(), collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
111 Assert.assertTrue(downloadedFile.exists());
112 Assert.assertEquals(file.getName(), downloadedFile.getName());
113 Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloadedFile), FileUtils.readFileToByteArray(file));
117 public void tearDown() throws Exception {
118 FileTestUtils.cleanDirectory(FILE_SPLIT_TEST_DIR);
119 FileTestUtils.cleanDirectory(FILE_DOWNLOAD_TEST_DIR);
122 private Collection prepareCollection() throws IOException {
123 // collection that will be returned by mocked collectionsApiClient
124 String filePath = "src/test/resources/org/arvados/client/api/client/collections-download-file.json";
125 File jsonFile = new File(filePath);
126 return MAPPER.readValue(jsonFile, Collection.class);
129 private ManifestStream prepareManifestStream() throws Exception {
130 // manifestStream that will be returned by mocked manifestDecoder
131 List<FileToken> fileTokens = new ArrayList<>();
132 fileTokens.add(new FileToken("0:1024:test-file1"));
133 fileTokens.add(new FileToken("1024:20480:test-file2"));
134 fileTokens.add(new FileToken("21504:1048576:test-file\\0403"));
136 KeepLocator keepLocator = new KeepLocator("163679d58edaadc28db769011728a72c+1070080+A3acf8c1fe582c265d2077702e4a7d74fcc03aba8@5aa4fdeb");
137 return new ManifestStream(".", Arrays.asList(keepLocator), fileTokens);
140 private byte[] prepareDataChunk(List<File> files) throws IOException {
141 File combinedFile = new File(FILE_SPLIT_TEST_DIR + Characters.SLASH + UUID.randomUUID());
142 FileMerge.merge(files, combinedFile);
143 return FileUtils.readFileToByteArray(combinedFile);