From b584bdc5d741d713413db74ce4486201ed4f9cc1 Mon Sep 17 00:00:00 2001 From: szlenkj Date: Fri, 26 Jan 2024 08:30:33 +0100 Subject: [PATCH] Download file with resume Arvados-DCO-1.1-Signed-off-by: Jakub Szlenk jakubszlenk@gmail.com --- .../client/api/client/KeepWebApiClient.java | 24 ++++++++++++++ .../client/logic/keep/FileDownloader.java | 31 +++++++++++++++++++ .../api/client/KeepWebApiClientTest.java | 28 ++++++++++++++++- 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java index 2c3168649f..2595956074 100644 --- a/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java +++ b/sdk/java-v2/src/main/java/org/arvados/client/api/client/KeepWebApiClient.java @@ -10,9 +10,13 @@ package org.arvados.client.api.client; import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; + import org.arvados.client.config.ConfigProvider; import java.io.File; +import java.io.IOException; import java.io.InputStream; public class KeepWebApiClient extends BaseApiClient { @@ -30,6 +34,26 @@ public class KeepWebApiClient extends BaseApiClient { return newFileCall(request); } + public byte[] downloadPartial(String collectionUuid, String filePathName, long offset) throws IOException { + Request.Builder builder = this.getRequestBuilder(); + if (offset > 0) { + builder.addHeader("Range", "bytes=" + offset + "-"); + } + Request request = builder.url(this.getUrlBuilder(collectionUuid, filePathName).build()).get().build(); + try (Response response = client.newCall(request).execute()) { + if (!response.isSuccessful()) { + throw new IOException("Failed to download file: " + response); + } + try (ResponseBody body = response.body()) { + if (body != null) { + return body.bytes(); + } else { + throw new IOException("Response body is null for request: " + request); + } + } + } + } + public String delete(String collectionUuid, String filePathName) { Request request = getRequestBuilder() .url(getUrlBuilder(collectionUuid, filePathName).build()) diff --git a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java index c1e8849e39..3fe4e6433a 100644 --- a/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java +++ b/sdk/java-v2/src/main/java/org/arvados/client/logic/keep/FileDownloader.java @@ -20,9 +20,12 @@ import org.arvados.client.logic.keep.exception.DownloadFolderAlreadyExistsExcept import org.arvados.client.logic.keep.exception.FileAlreadyExistsException; import org.slf4j.Logger; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -70,6 +73,34 @@ public class FileDownloader { return downloadedFile; } + public File downloadFileWithResume(String collectionUuid, String fileName, String pathToDownloadFolder, long offset, int bufferSize) throws IOException { + if (bufferSize <= 0) { + throw new IllegalArgumentException("Buffer size must be greater than 0"); + } + + File destinationFile = new File(pathToDownloadFolder, fileName); + + if (!destinationFile.exists()) { + boolean isCreated = destinationFile.createNewFile(); + if (!isCreated) { + throw new IOException("Failed to create new file: " + destinationFile.getAbsolutePath()); + } + } + + try (RandomAccessFile outputFile = new RandomAccessFile(destinationFile, "rw")) { + outputFile.seek(offset); + + byte[] buffer = new byte[bufferSize]; + int bytesRead; + InputStream inputStream = new ByteArrayInputStream(keepWebApiClient.downloadPartial(collectionUuid, fileName, offset)); + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputFile.write(buffer, 0, bytesRead); + } + } + + return destinationFile; + } + public List downloadFilesFromCollectionUsingKeepWeb(String collectionUuid, String pathToDownloadFolder) { String collectionTargetDir = setTargetDirectory(collectionUuid, pathToDownloadFolder).getAbsolutePath(); List fileTokens = listFileInfoFromCollection(collectionUuid); diff --git a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java index 07b7b25339..e796c69032 100644 --- a/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java +++ b/sdk/java-v2/src/test/java/org/arvados/client/api/client/KeepWebApiClientTest.java @@ -13,12 +13,17 @@ import org.junit.Test; import java.io.File; import java.nio.file.Files; +import okhttp3.mockwebserver.MockResponse; +import okio.Buffer; + import static org.arvados.client.test.utils.ApiClientTestUtils.getResponse; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest { - private KeepWebApiClient client = new KeepWebApiClient(CONFIG); + private final KeepWebApiClient client = new KeepWebApiClient(CONFIG); @Test public void uploadFile() throws Exception { @@ -36,4 +41,25 @@ public class KeepWebApiClientTest extends ArvadosClientMockedWebServerTest { assertThat(uploadResponse).isEqualTo("Created"); } + @Test + public void downloadPartialIsPerformedSuccessfully() throws Exception { + // given + String collectionUuid = "some-collection-uuid"; + String filePathName = "sample-file-path"; + long offset = 1024; + + byte[] expectedData = "test data".getBytes(); + + try (Buffer buffer = new Buffer().write(expectedData)) { + server.enqueue(new MockResponse().setBody(buffer)); + + // when + byte[] actualData = client.downloadPartial(collectionUuid, filePathName, offset); + + // then + assertNotNull(actualData); + assertEquals(new String(expectedData), new String(actualData)); + } + } + } -- 2.30.2