sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java

   1 /*
   2  * Copyright (C) The Arvados Authors. All rights reserved.
   3  *
   4  * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
   5  *
   6  */
   7
   8 package org.arvados.client.logic.keep;
   9
  10 import com.fasterxml.jackson.databind.ObjectMapper;
  11 import org.arvados.client.api.client.CollectionsApiClient;
  12 import org.arvados.client.api.client.KeepWebApiClient;
  13 import org.arvados.client.api.model.Collection;
  14 import org.arvados.client.common.Characters;
  15 import org.arvados.client.logic.collection.FileToken;
  16 import org.arvados.client.logic.collection.ManifestDecoder;
  17 import org.arvados.client.logic.collection.ManifestStream;
  18 import org.arvados.client.test.utils.FileTestUtils;
  19 import org.arvados.client.utils.FileMerge;
  20 import org.apache.commons.io.FileUtils;
  21 import org.junit.After;
  22 import org.junit.Before;
  23 import org.junit.Test;
  24 import org.junit.runner.RunWith;
  25 import org.mockito.InjectMocks;
  26 import org.mockito.Mock;
  27 import org.mockito.junit.MockitoJUnitRunner;
  28
  29 import java.io.ByteArrayInputStream;
  30 import java.io.File;
  31 import java.io.IOException;
  32 import java.io.InputStream;
  33 import java.nio.file.Files;
  34 import java.util.ArrayList;
  35 import java.util.Arrays;
  36 import java.util.List;
  37 import java.util.UUID;
  38
  39 import static org.arvados.client.test.utils.FileTestUtils.*;
  40 import static org.assertj.core.api.Assertions.assertThat;
  41 import static org.junit.Assert.assertArrayEquals;
  42 import static org.junit.Assert.assertEquals;
  43 import static org.junit.Assert.assertNotNull;
  44 import static org.junit.Assert.assertTrue;
  45 import static org.mockito.Mockito.when;
  46
  47 @RunWith(MockitoJUnitRunner.class)
  48 public class FileDownloaderTest {
  49
  50     static final ObjectMapper MAPPER = new ObjectMapper().findAndRegisterModules();
  51     private Collection collectionToDownload;
  52     private ManifestStream manifestStream;
  53
  54     @Mock
  55     private CollectionsApiClient collectionsApiClient;
  56     @Mock
  57     private KeepClient keepClient;
  58     @Mock
  59     private KeepWebApiClient keepWebApiClient;
  60     @Mock
  61     private ManifestDecoder manifestDecoder;
  62     @InjectMocks
  63     private FileDownloader fileDownloader;
  64
  65     @Before
  66     public void setUp() throws Exception {
  67         FileTestUtils.createDirectory(FILE_SPLIT_TEST_DIR);
  68         FileTestUtils.createDirectory(FILE_DOWNLOAD_TEST_DIR);
  69
  70         collectionToDownload = prepareCollection();
  71         manifestStream = prepareManifestStream();
  72     }
  73
  74     @Test
  75     public void downloadingAllFilesFromCollectionWorksProperly() throws Exception {
  76         // given
  77         List<File> files = generatePredefinedFiles();
  78         byte[] dataChunk = prepareDataChunk(files);
  79
  80         //having
  81         when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
  82         when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
  83         when(keepClient.getDataChunk(manifestStream.getKeepLocators().get(0))).thenReturn(dataChunk);
  84
  85         //when
  86         List<File> downloadedFiles = fileDownloader.downloadFilesFromCollection(collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
  87
  88         //then
  89         assertEquals(3, downloadedFiles.size()); // 3 files downloaded
  90
  91         File collectionDir = new File(FILE_DOWNLOAD_TEST_DIR + Characters.SLASH + collectionToDownload.getUuid());
  92         assertTrue(collectionDir.exists()); // collection directory created
  93
  94         // 3 files correctly saved
  95         assertThat(downloadedFiles).allMatch(File::exists);
  96
  97         for(int i = 0; i < downloadedFiles.size(); i ++) {
  98             File downloaded = new File(collectionDir + Characters.SLASH + files.get(i).getName());
  99             assertArrayEquals(FileUtils.readFileToByteArray(downloaded), FileUtils.readFileToByteArray(files.get(i)));
 100         }
 101     }
 102
 103     @Test
 104     public void downloadingSingleFileFromKeepWebWorksCorrectly() throws Exception{
 105         //given
 106         File file = generatePredefinedFiles().get(0);
 107
 108         //having
 109         when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
 110         when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
 111         when(keepWebApiClient.download(collectionToDownload.getUuid(), file.getName())).thenReturn(FileUtils.readFileToByteArray(file));
 112
 113         //when
 114         File downloadedFile = fileDownloader.downloadSingleFileUsingKeepWeb(file.getName(), collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
 115
 116         //then
 117         assertTrue(downloadedFile.exists());
 118         assertEquals(file.getName(), downloadedFile.getName());
 119         assertArrayEquals(FileUtils.readFileToByteArray(downloadedFile), FileUtils.readFileToByteArray(file));
 120     }
 121
 122     @Test
 123     public void testDownloadFileWithResume() throws Exception {
 124         //given
 125         String collectionUuid = "some-collection-uuid";
 126         String expectedDataString = "testData";
 127         String fileName = "sample-file-name";
 128         long start = 0;
 129         Long end = null;
 130
 131         InputStream inputStream = new ByteArrayInputStream(expectedDataString.getBytes());
 132
 133         when(keepWebApiClient.get(collectionUuid, fileName, start, end)).thenReturn(inputStream);
 134
 135         //when
 136         File downloadedFile = fileDownloader.downloadFileWithResume(collectionUuid, fileName, FILE_DOWNLOAD_TEST_DIR, start, end);
 137
 138         //then
 139         assertNotNull(downloadedFile);
 140         assertTrue(downloadedFile.exists());
 141         String actualDataString = Files.readString(downloadedFile.toPath());
 142         assertEquals("The content of the file does not match the expected data.", expectedDataString, actualDataString);
 143     }
 144
 145     @After
 146     public void tearDown() throws Exception {
 147         FileTestUtils.cleanDirectory(FILE_SPLIT_TEST_DIR);
 148         FileTestUtils.cleanDirectory(FILE_DOWNLOAD_TEST_DIR);
 149     }
 150
 151     private Collection prepareCollection() throws IOException {
 152         // collection that will be returned by mocked collectionsApiClient
 153         String filePath = "src/test/resources/org/arvados/client/api/client/collections-download-file.json";
 154         File jsonFile = new File(filePath);
 155         return MAPPER.readValue(jsonFile, Collection.class);
 156     }
 157
 158     private ManifestStream prepareManifestStream() throws Exception {
 159         // manifestStream that will be returned by mocked manifestDecoder
 160         List<FileToken> fileTokens = new ArrayList<>();
 161         fileTokens.add(new FileToken("0:1024:test-file1"));
 162         fileTokens.add(new FileToken("1024:20480:test-file2"));
 163         fileTokens.add(new FileToken("21504:1048576:test-file\\0403"));
 164
 165         KeepLocator keepLocator = new KeepLocator("163679d58edaadc28db769011728a72c+1070080+A3acf8c1fe582c265d2077702e4a7d74fcc03aba8@5aa4fdeb");
 166         return new ManifestStream(".", Arrays.asList(keepLocator), fileTokens);
 167     }
 168
 169     private byte[] prepareDataChunk(List<File> files) throws IOException {
 170         File combinedFile = new File(FILE_SPLIT_TEST_DIR + Characters.SLASH + UUID.randomUUID());
 171         FileMerge.merge(files, combinedFile);
 172         return FileUtils.readFileToByteArray(combinedFile);
 173     }
 174 }