From f1bbac648067f651d408d3cad39fd31d9a36354d Mon Sep 17 00:00:00 2001 From: Daniel Kos Date: Tue, 7 Aug 2018 00:31:54 +0200 Subject: [PATCH] Add multiple files uploading Feature #13856 Arvados-DCO-1.1-Signed-off-by: Daniel Kos --- src/components/file-upload/file-upload.tsx | 9 +- .../collection-files-service.ts | 1 - .../collection-manifest-parser.ts | 4 +- .../collection-service/collection-service.ts | 86 +++++++++++++++---- .../creator/collection-creator-action.ts | 5 +- .../create-collection-dialog.tsx | 5 +- 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/src/components/file-upload/file-upload.tsx b/src/components/file-upload/file-upload.tsx index 5fb28428..8c6e04a9 100644 --- a/src/components/file-upload/file-upload.tsx +++ b/src/components/file-upload/file-upload.tsx @@ -41,13 +41,14 @@ export const FileUpload = withStyles(styles)( Upload data onDrop(files)}> - - + + - Drag and drop data or browse + Drag and drop data or click to browse - + + {files.map((f, idx) => diff --git a/src/services/collection-files-service/collection-files-service.ts b/src/services/collection-files-service/collection-files-service.ts index dfeed0b7..5e6891c8 100644 --- a/src/services/collection-files-service/collection-files-service.ts +++ b/src/services/collection-files-service/collection-files-service.ts @@ -5,7 +5,6 @@ import { CollectionService } from "../collection-service/collection-service"; import { parseKeepManifestText, stringifyKeepManifest } from "./collection-manifest-parser"; import { mapManifestToCollectionFilesTree } from "./collection-manifest-mapper"; -import { CollectionFile } from "../../models/collection-file"; import { CommonResourceService } from "../../common/api/common-resource-service"; import * as _ from "lodash"; diff --git a/src/services/collection-files-service/collection-manifest-parser.ts b/src/services/collection-files-service/collection-manifest-parser.ts index bed26783..b0fc55a4 100644 --- a/src/services/collection-files-service/collection-manifest-parser.ts +++ b/src/services/collection-files-service/collection-manifest-parser.ts @@ -7,7 +7,7 @@ import { KeepManifestStream, KeepManifestStreamFile, KeepManifest } from "../../ /** * Documentation [http://doc.arvados.org/api/storage.html](http://doc.arvados.org/api/storage.html) */ -export const parseKeepManifestText = (text: string) => +export const parseKeepManifestText: (text: string) => KeepManifestStream[] = (text: string) => text .split(/\n/) .filter(streamText => streamText.length > 0) @@ -52,4 +52,4 @@ const parseFile = (token: string): KeepManifestStreamFile => { }; const stringifyFile = (file: KeepManifestStreamFile) => - `${file.position}:${file.size}:${file.name}`; \ No newline at end of file + `${file.position}:${file.size}:${file.name}`; diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts index 88f3daef..cf2d53e8 100644 --- a/src/services/collection-service/collection-service.ts +++ b/src/services/collection-service/collection-service.ts @@ -7,36 +7,86 @@ import { CollectionResource } from "../../models/collection"; import axios, { AxiosInstance } from "axios"; import { KeepService } from "../keep-service/keep-service"; import { FilterBuilder } from "../../common/api/filter-builder"; +import { CollectionFile, CollectionFileType, createCollectionFile } from "../../models/collection-file"; +import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser"; +import * as _ from "lodash"; +import { KeepManifestStream } from "../../models/keep-manifest"; export class CollectionService extends CommonResourceService { constructor(serverApi: AxiosInstance, private keepService: KeepService) { super(serverApi, "collections"); } - uploadFiles(files: File[]) { - console.log("Uploading files", files); - + uploadFile(keepServiceHost: string, file: File, fileIdx = 0): Promise { const fd = new FormData(); - fd.append("filters", `[["service_type","=","proxy"]]`); - fd.append("_method", "GET"); + fd.append(`file_${fileIdx}`, file); - const filters = new FilterBuilder(); - filters.addEqual("service_type", "proxy"); + return axios.post(keepServiceHost, fd, { + onUploadProgress: (e: ProgressEvent) => { + console.log(`${e.loaded} / ${e.total}`); + } + }).then(data => createCollectionFile({ + id: data.data, + name: file.name, + size: file.size + })); + } - return this.keepService.list({ filters }).then(data => { - console.log(data); + private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise { + const collection = await this.get(collectionUuid); + const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText); - const serviceHost = (data.items[0].serviceSslFlag ? "https://" : "http://") + data.items[0].serviceHost + ":" + data.items[0].servicePort; - console.log("Servicehost", serviceHost); + files.forEach(f => { + let kms = manifest.find(stream => stream.name === f.path); + if (!kms) { + kms = { + files: [], + locators: [], + name: f.path + }; + manifest.push(kms); + } + kms.locators.push(f.id); + const len = kms.files.length; + const nextPos = len > 0 + ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size + : 0; + kms.files.push({ + name: f.name, + position: nextPos.toString(), + size: f.size + }); + }); - const fd = new FormData(); - files.forEach((f, idx) => fd.append(`file_${idx}`, f)); + console.log(manifest); - axios.post(serviceHost, fd, { - onUploadProgress: (e: ProgressEvent) => { - console.log(`${e.loaded} / ${e.total}`); - } - }); + const manifestText = stringifyKeepManifest(manifest); + const data = { ...collection, manifestText }; + return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data)); + } + + uploadFiles(collectionUuid: string, files: File[]) { + console.log("Uploading files", files); + + const filters = FilterBuilder.create() + .addEqual("service_type", "proxy"); + + return this.keepService.list({ filters }).then(data => { + if (data.items && data.items.length > 0) { + const serviceHost = + (data.items[0].serviceSslFlag ? "https://" : "http://") + + data.items[0].serviceHost + + ":" + data.items[0].servicePort; + + console.log("Servicehost", serviceHost); + + const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx)); + Promise.all(files$).then(values => { + this.updateManifest(collectionUuid, values).then(() => { + console.log("Upload done!"); + }); + }); + } }); } } diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts index b06cf0f5..3afe0e92 100644 --- a/src/store/collections/creator/collection-creator-action.ts +++ b/src/store/collections/creator/collection-creator-action.ts @@ -26,7 +26,10 @@ export const createCollection = (collection: Partial) => dispatch(collectionCreateActions.CREATE_COLLECTION(collectiontData)); return services.collectionService .create(collectiontData) - .then(collection => dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection))); + .then(collection => { + dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection)); + return collection; + }); }; export type CollectionCreateAction = UnionOf; diff --git a/src/views-components/create-collection-dialog/create-collection-dialog.tsx b/src/views-components/create-collection-dialog/create-collection-dialog.tsx index 281bd54e..8711c5fa 100644 --- a/src/views-components/create-collection-dialog/create-collection-dialog.tsx +++ b/src/views-components/create-collection-dialog/create-collection-dialog.tsx @@ -13,6 +13,7 @@ import { dataExplorerActions } from "../../store/data-explorer/data-explorer-act import { PROJECT_PANEL_ID } from "../../views/project-panel/project-panel"; import { snackbarActions } from "../../store/snackbar/snackbar-actions"; import { ServiceRepository } from "../../services/services"; +import { CollectionResource } from "../../models/collection"; const mapStateToProps = (state: RootState) => ({ open: state.collections.creator.opened @@ -32,12 +33,12 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ const addCollection = (data: { name: string, description: string, files: File[] }) => (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => { - return dispatch(createCollection(data)).then(() => { + return dispatch(createCollection(data)).then((collection: CollectionResource) => { dispatch(snackbarActions.OPEN_SNACKBAR({ message: "Collection has been successfully created.", hideDuration: 2000 })); - services.collectionService.uploadFiles(data.files).then(() => { + services.collectionService.uploadFiles(collection.uuid, data.files).then(() => { dispatch(dataExplorerActions.REQUEST_ITEMS({ id: PROJECT_PANEL_ID })); }); }); -- 2.30.2