sdk/java-v2/src/test/java/org/arvados/client/logic/keep/FileDownloaderTest.java

   1 /*
   2  * Copyright (C) The Arvados Authors. All rights reserved.
   3  *
   4  * SPDX-License-Identifier: AGPL-3.0 OR Apache-2.0
   5  *
   6  */
   7
   8 package org.arvados.client.logic.keep;
   9
  10 import com.fasterxml.jackson.databind.ObjectMapper;
  11 import org.arvados.client.api.client.CollectionsApiClient;
  12 import org.arvados.client.api.client.KeepWebApiClient;
  13 import org.arvados.client.api.model.Collection;
  14 import org.arvados.client.common.Characters;
  15 import org.arvados.client.logic.collection.FileToken;
  16 import org.arvados.client.logic.collection.ManifestDecoder;
  17 import org.arvados.client.logic.collection.ManifestStream;
  18 import org.arvados.client.test.utils.FileTestUtils;
  19 import org.arvados.client.utils.FileMerge;
  20 import org.apache.commons.io.FileUtils;
  21 import org.junit.After;
  22 import org.junit.Assert;
  23 import org.junit.Before;
  24 import org.junit.Test;
  25 import org.junit.runner.RunWith;
  26 import org.mockito.InjectMocks;
  27 import org.mockito.Mock;
  28 import org.mockito.junit.MockitoJUnitRunner;
  29
  30 import java.io.File;
  31 import java.io.IOException;
  32 import java.util.ArrayList;
  33 import java.util.Arrays;
  34 import java.util.List;
  35 import java.util.UUID;
  36
  37 import static org.arvados.client.test.utils.FileTestUtils.*;
  38 import static org.assertj.core.api.Assertions.assertThat;
  39 import static org.mockito.Mockito.when;
  40
  41 @RunWith(MockitoJUnitRunner.class)
  42 public class FileDownloaderTest {
  43
  44     static final ObjectMapper MAPPER = new ObjectMapper().findAndRegisterModules();
  45     private Collection collectionToDownload;
  46     private ManifestStream manifestStream;
  47
  48     @Mock
  49     private CollectionsApiClient collectionsApiClient;
  50     @Mock
  51     private KeepClient keepClient;
  52     @Mock
  53     private KeepWebApiClient keepWebApiClient;
  54     @Mock
  55     private ManifestDecoder manifestDecoder;
  56     @InjectMocks
  57     private FileDownloader fileDownloader;
  58
  59     @Before
  60     public void setUp() throws Exception {
  61         FileTestUtils.createDirectory(FILE_SPLIT_TEST_DIR);
  62         FileTestUtils.createDirectory(FILE_DOWNLOAD_TEST_DIR);
  63
  64         collectionToDownload = prepareCollection();
  65         manifestStream = prepareManifestStream();
  66     }
  67
  68     @Test
  69     public void downloadingAllFilesFromCollectionWorksProperly() throws Exception {
  70         // given
  71         List<File> files = generatePredefinedFiles();
  72         byte[] dataChunk = prepareDataChunk(files);
  73
  74         //having
  75         when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
  76         when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
  77         when(keepClient.getDataChunk(manifestStream.getKeepLocators().get(0))).thenReturn(dataChunk);
  78
  79         //when
  80         List<File> downloadedFiles = fileDownloader.downloadFilesFromCollection(collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
  81
  82         //then
  83         Assert.assertEquals(3, downloadedFiles.size()); // 3 files downloaded
  84
  85         File collectionDir = new File(FILE_DOWNLOAD_TEST_DIR + Characters.SLASH + collectionToDownload.getUuid());
  86         Assert.assertTrue(collectionDir.exists()); // collection directory created
  87
  88         // 3 files correctly saved
  89         assertThat(downloadedFiles).allMatch(File::exists);
  90
  91         for(int i = 0; i < downloadedFiles.size(); i ++) {
  92             File downloaded = new File(collectionDir + Characters.SLASH + files.get(i).getName());
  93             Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloaded), FileUtils.readFileToByteArray(files.get(i)));
  94         }
  95     }
  96
  97     @Test
  98     public void downloadingSingleFileFromKeepWebWorksCorrectly() throws Exception{
  99         //given
 100         File file = generatePredefinedFiles().get(0);
 101
 102         //having
 103         when(collectionsApiClient.get(collectionToDownload.getUuid())).thenReturn(collectionToDownload);
 104         when(manifestDecoder.decode(collectionToDownload.getManifestText())).thenReturn(Arrays.asList(manifestStream));
 105         when(keepWebApiClient.download(collectionToDownload.getUuid(), file.getName())).thenReturn(FileUtils.readFileToByteArray(file));
 106
 107         //when
 108         File downloadedFile = fileDownloader.downloadSingleFileUsingKeepWeb(file.getName(), collectionToDownload.getUuid(), FILE_DOWNLOAD_TEST_DIR);
 109
 110         //then
 111         Assert.assertTrue(downloadedFile.exists());
 112         Assert.assertEquals(file.getName(), downloadedFile.getName());
 113         Assert.assertArrayEquals(FileUtils.readFileToByteArray(downloadedFile), FileUtils.readFileToByteArray(file));
 114     }
 115
 116     @After
 117     public void tearDown() throws Exception {
 118         FileTestUtils.cleanDirectory(FILE_SPLIT_TEST_DIR);
 119         FileTestUtils.cleanDirectory(FILE_DOWNLOAD_TEST_DIR);
 120     }
 121
 122     private Collection prepareCollection() throws IOException {
 123         // collection that will be returned by mocked collectionsApiClient
 124         String filePath = "src/test/resources/org/arvados/client/api/client/collections-download-file.json";
 125         File jsonFile = new File(filePath);
 126         return MAPPER.readValue(jsonFile, Collection.class);
 127     }
 128
 129     private ManifestStream prepareManifestStream() throws Exception {
 130         // manifestStream that will be returned by mocked manifestDecoder
 131         List<FileToken> fileTokens = new ArrayList<>();
 132         fileTokens.add(new FileToken("0:1024:test-file1"));
 133         fileTokens.add(new FileToken("1024:20480:test-file2"));
 134         fileTokens.add(new FileToken("21504:1048576:test-file\\0403"));
 135
 136         KeepLocator keepLocator = new KeepLocator("163679d58edaadc28db769011728a72c+1070080+A3acf8c1fe582c265d2077702e4a7d74fcc03aba8@5aa4fdeb");
 137         return new ManifestStream(".", Arrays.asList(keepLocator), fileTokens);
 138     }
 139
 140     private byte[] prepareDataChunk(List<File> files) throws IOException {
 141         File combinedFile = new File(FILE_SPLIT_TEST_DIR + Characters.SLASH + UUID.randomUUID());
 142         FileMerge.merge(files, combinedFile);
 143         return FileUtils.readFileToByteArray(combinedFile);
 144     }
 145 }