From 961394c0876cdc07f2195d3bc1843a5c9a6fa950 Mon Sep 17 00:00:00 2001 From: Daniel Kos Date: Tue, 7 Aug 2018 11:33:12 +0200 Subject: [PATCH] Fix correct bytes not being sent, fix showing upload progress and speed Feature #13856 Arvados-DCO-1.1-Signed-off-by: Daniel Kos --- src/common/formatters.ts | 14 ++++- src/components/file-upload/file-upload.tsx | 53 ++++++++++------ .../collection-service/collection-service.ts | 60 ++++++++++++------- .../creator/collection-creator-action.ts | 12 +++- .../uploader/collection-uploader-actions.ts | 15 ++++- .../uploader/collection-uploader-reducer.ts | 42 +++++++++---- .../create-collection-dialog.tsx | 18 ++---- .../dialog-collection-create.tsx | 14 ++--- 8 files changed, 153 insertions(+), 75 deletions(-) diff --git a/src/common/formatters.ts b/src/common/formatters.ts index 38ef0223..49e06905 100644 --- a/src/common/formatters.ts +++ b/src/common/formatters.ts @@ -19,6 +19,18 @@ export const formatFileSize = (size?: number) => { return ""; }; +export const formatProgress = (loaded: number, total: number) => { + const progress = loaded >= 0 && total > 0 ? loaded * 100 / total : 0; + return `${progress.toFixed(2)}%`; +}; + +export function formatUploadSpeed(prevLoaded: number, loaded: number, prevTime: number, currentTime: number) { + const speed = loaded > prevLoaded && currentTime > prevTime + ? (loaded - prevLoaded) / (currentTime - prevTime) + : 0; + return `${(speed / 1000).toFixed(2)} KB/s`; +} + const FILE_SIZES = [ { base: 1000000000000, @@ -40,4 +52,4 @@ const FILE_SIZES = [ base: 1, unit: "B" } -]; \ No newline at end of file +]; diff --git a/src/components/file-upload/file-upload.tsx b/src/components/file-upload/file-upload.tsx index 8c6e04a9..aa3c0e96 100644 --- a/src/components/file-upload/file-upload.tsx +++ b/src/components/file-upload/file-upload.tsx @@ -3,11 +3,18 @@ // SPDX-License-Identifier: AGPL-3.0 import * as React from 'react'; -import { Grid, List, ListItem, ListItemText, StyleRulesCallback, Typography, WithStyles } from '@material-ui/core'; +import { + Grid, + StyleRulesCallback, + Table, TableBody, TableCell, TableHead, TableRow, + Typography, + WithStyles +} from '@material-ui/core'; import { withStyles } from '@material-ui/core'; import Dropzone from 'react-dropzone'; import { CloudUploadIcon } from "../icon/icon"; -import { formatFileSize } from "../../common/formatters"; +import { formatFileSize, formatProgress, formatUploadSpeed } from "../../common/formatters"; +import { UploadFile } from "../../store/collections/uploader/collection-uploader-actions"; type CssRules = "root" | "dropzone" | "container" | "uploadIcon"; @@ -30,7 +37,7 @@ const styles: StyleRulesCallback = theme => ({ }); interface FileUploadProps { - files: File[]; + files: UploadFile[]; onDrop: (files: File[]) => void; } @@ -41,24 +48,36 @@ export const FileUpload = withStyles(styles)( Upload data onDrop(files)}> - - + {files.length === 0 && + + Drag and drop data or click to browse - - - - {files.map((f, idx) => - - - )} - - - + } + {files.length > 0 && + + + + File name + File size + Upload speed + Upload progress + + + + {files.map(f => + + {f.file.name} + {formatFileSize(f.file.size)} + {formatUploadSpeed(f.prevLoaded, f.loaded, f.prevTime, f.currentTime)} + {formatProgress(f.loaded, f.total)} + + )} + +
+ } ); diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts index cf2d53e8..4d750362 100644 --- a/src/services/collection-service/collection-service.ts +++ b/src/services/collection-service/collection-service.ts @@ -7,29 +7,47 @@ import { CollectionResource } from "../../models/collection"; import axios, { AxiosInstance } from "axios"; import { KeepService } from "../keep-service/keep-service"; import { FilterBuilder } from "../../common/api/filter-builder"; -import { CollectionFile, CollectionFileType, createCollectionFile } from "../../models/collection-file"; +import { CollectionFile, createCollectionFile } from "../../models/collection-file"; import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser"; import * as _ from "lodash"; import { KeepManifestStream } from "../../models/keep-manifest"; +export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void; + export class CollectionService extends CommonResourceService { constructor(serverApi: AxiosInstance, private keepService: KeepService) { super(serverApi, "collections"); } - uploadFile(keepServiceHost: string, file: File, fileIdx = 0): Promise { - const fd = new FormData(); - fd.append(`file_${fileIdx}`, file); + private readFile(file: File): Promise { + return new Promise(resolve => { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as ArrayBuffer); + }; - return axios.post(keepServiceHost, fd, { - onUploadProgress: (e: ProgressEvent) => { - console.log(`${e.loaded} / ${e.total}`); - } - }).then(data => createCollectionFile({ - id: data.data, - name: file.name, - size: file.size - })); + reader.readAsArrayBuffer(file); + }); + } + + private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise { + return this.readFile(file).then(content => { + return axios.post(keepServiceHost, content, { + headers: { + 'Content-Type': 'text/octet-stream' + }, + onUploadProgress: (e: ProgressEvent) => { + if (onProgress) { + onProgress(fileId, e.loaded, e.total, Date.now()); + } + console.log(`${e.loaded} / ${e.total}`); + } + }).then(data => createCollectionFile({ + id: data.data, + name: file.name, + size: file.size + })); + }); } private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise { @@ -65,9 +83,7 @@ export class CollectionService extends CommonResourceService return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data)); } - uploadFiles(collectionUuid: string, files: File[]) { - console.log("Uploading files", files); - + uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise { const filters = FilterBuilder.create() .addEqual("service_type", "proxy"); @@ -78,14 +94,14 @@ export class CollectionService extends CommonResourceService data.items[0].serviceHost + ":" + data.items[0].servicePort; - console.log("Servicehost", serviceHost); + console.log("serviceHost", serviceHost); - const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx)); - Promise.all(files$).then(values => { - this.updateManifest(collectionUuid, values).then(() => { - console.log("Upload done!"); - }); + const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress)); + return Promise.all(files$).then(values => { + return this.updateManifest(collectionUuid, values); }); + } else { + return Promise.reject("Missing keep service host"); } }); } diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts index 3afe0e92..023e5be6 100644 --- a/src/store/collections/creator/collection-creator-action.ts +++ b/src/store/collections/creator/collection-creator-action.ts @@ -8,6 +8,7 @@ import { Dispatch } from "redux"; import { RootState } from "../../store"; import { CollectionResource } from '../../../models/collection'; import { ServiceRepository } from "../../../services/services"; +import { collectionUploaderActions } from "../uploader/collection-uploader-actions"; export const collectionCreateActions = unionize({ OPEN_COLLECTION_CREATOR: ofType<{ ownerUuid: string }>(), @@ -19,7 +20,7 @@ export const collectionCreateActions = unionize({ value: 'payload' }); -export const createCollection = (collection: Partial) => +export const createCollection = (collection: Partial, files: File[]) => (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => { const { ownerUuid } = getState().collections.creator; const collectiontData = { ownerUuid, ...collection }; @@ -27,7 +28,14 @@ export const createCollection = (collection: Partial) => return services.collectionService .create(collectiontData) .then(collection => { - dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection)); + dispatch(collectionUploaderActions.START_UPLOAD()); + services.collectionService.uploadFiles(collection.uuid, files, + (fileId, loaded, total, currentTime) => { + dispatch(collectionUploaderActions.SET_UPLOAD_PROGRESS({ fileId, loaded, total, currentTime })); + }) + .then(collection => { + dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection)); + }); return collection; }); }; diff --git a/src/store/collections/uploader/collection-uploader-actions.ts b/src/store/collections/uploader/collection-uploader-actions.ts index 0b9aeb99..7c85d740 100644 --- a/src/store/collections/uploader/collection-uploader-actions.ts +++ b/src/store/collections/uploader/collection-uploader-actions.ts @@ -4,10 +4,21 @@ import { default as unionize, ofType, UnionOf } from "unionize"; +export interface UploadFile { + id: number; + file: File; + prevLoaded: number; + loaded: number; + total: number; + startTime: number; + prevTime: number; + currentTime: number; +} + export const collectionUploaderActions = unionize({ SET_UPLOAD_FILES: ofType(), - START_UPLOADING: ofType<{}>(), - UPDATE_UPLOAD_PROGRESS: ofType<{}>() + START_UPLOAD: ofType(), + SET_UPLOAD_PROGRESS: ofType<{ fileId: number, loaded: number, total: number, currentTime: number }>() }, { tag: 'type', value: 'payload' diff --git a/src/store/collections/uploader/collection-uploader-reducer.ts b/src/store/collections/uploader/collection-uploader-reducer.ts index 05735d65..5b24d2c4 100644 --- a/src/store/collections/uploader/collection-uploader-reducer.ts +++ b/src/store/collections/uploader/collection-uploader-reducer.ts @@ -2,23 +2,41 @@ // // SPDX-License-Identifier: AGPL-3.0 -import { CollectionUploaderAction, collectionUploaderActions } from "./collection-uploader-actions"; -import { CollectionUploadFile } from "../../../models/collection-file"; +import { CollectionUploaderAction, collectionUploaderActions, UploadFile } from "./collection-uploader-actions"; +import * as _ from 'lodash'; -export interface CollectionUploaderState { - files: File[]; -} +export type CollectionUploaderState = UploadFile[]; -const initialState: CollectionUploaderState = { - files: [] -}; +const initialState: CollectionUploaderState = []; export const collectionUploaderReducer = (state: CollectionUploaderState = initialState, action: CollectionUploaderAction) => { return collectionUploaderActions.match(action, { - SET_UPLOAD_FILES: (files) => ({ - ...state, - files - }), + SET_UPLOAD_FILES: files => files.map((f, idx) => ({ + id: idx, + file: f, + prevLoaded: 0, + loaded: 0, + total: 0, + startTime: 0, + prevTime: 0, + currentTime: 0 + })), + START_UPLOAD: () => { + const startTime = Date.now(); + return state.map(f => ({...f, startTime, prevTime: startTime})); + }, + SET_UPLOAD_PROGRESS: ({ fileId, loaded, total, currentTime }) => { + const files = _.cloneDeep(state); + const f = files.find(f => f.id === fileId); + if (f) { + f.prevLoaded = f.loaded; + f.loaded = loaded; + f.total = total; + f.prevTime = f.currentTime; + f.currentTime = currentTime; + } + return files; + }, default: () => state }); }; diff --git a/src/views-components/create-collection-dialog/create-collection-dialog.tsx b/src/views-components/create-collection-dialog/create-collection-dialog.tsx index 8711c5fa..3f8cc07c 100644 --- a/src/views-components/create-collection-dialog/create-collection-dialog.tsx +++ b/src/views-components/create-collection-dialog/create-collection-dialog.tsx @@ -9,11 +9,8 @@ import { SubmissionError } from "redux-form"; import { RootState } from "../../store/store"; import { DialogCollectionCreate } from "../dialog-create/dialog-collection-create"; import { collectionCreateActions, createCollection } from "../../store/collections/creator/collection-creator-action"; -import { dataExplorerActions } from "../../store/data-explorer/data-explorer-action"; -import { PROJECT_PANEL_ID } from "../../views/project-panel/project-panel"; import { snackbarActions } from "../../store/snackbar/snackbar-actions"; -import { ServiceRepository } from "../../services/services"; -import { CollectionResource } from "../../models/collection"; +import { UploadFile } from "../../store/collections/uploader/collection-uploader-actions"; const mapStateToProps = (state: RootState) => ({ open: state.collections.creator.opened @@ -23,24 +20,21 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ handleClose: () => { dispatch(collectionCreateActions.CLOSE_COLLECTION_CREATOR()); }, - onSubmit: (data: { name: string, description: string, files: File[] }) => { - return dispatch(addCollection(data)) + onSubmit: (data: { name: string, description: string }, files: UploadFile[]) => { + return dispatch(addCollection(data, files.map(f => f.file))) .catch((e: any) => { throw new SubmissionError({ name: e.errors.join("").includes("UniqueViolation") ? "Collection with this name already exists." : "" }); }); } }); -const addCollection = (data: { name: string, description: string, files: File[] }) => - (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => { - return dispatch(createCollection(data)).then((collection: CollectionResource) => { +const addCollection = (data: { name: string, description: string }, files: File[]) => + (dispatch: Dispatch) => { + return dispatch(createCollection(data, files)).then(() => { dispatch(snackbarActions.OPEN_SNACKBAR({ message: "Collection has been successfully created.", hideDuration: 2000 })); - services.collectionService.uploadFiles(collection.uuid, data.files).then(() => { - dispatch(dataExplorerActions.REQUEST_ITEMS({ id: PROJECT_PANEL_ID })); - }); }); }; diff --git a/src/views-components/dialog-create/dialog-collection-create.tsx b/src/views-components/dialog-create/dialog-collection-create.tsx index a0cbcba5..32fc6572 100644 --- a/src/views-components/dialog-create/dialog-collection-create.tsx +++ b/src/views-components/dialog-create/dialog-collection-create.tsx @@ -16,7 +16,7 @@ import { COLLECTION_NAME_VALIDATION, COLLECTION_DESCRIPTION_VALIDATION } from '. import { FileUpload } from "../../components/file-upload/file-upload"; import { connect, DispatchProp } from "react-redux"; import { RootState } from "../../store/store"; -import { collectionUploaderActions } from "../../store/collections/uploader/collection-uploader-actions"; +import { collectionUploaderActions, UploadFile } from "../../store/collections/uploader/collection-uploader-actions"; type CssRules = "button" | "lastButton" | "formContainer" | "textField" | "createProgress" | "dialogActions"; @@ -48,12 +48,12 @@ const styles: StyleRulesCallback = theme => ({ interface DialogCollectionCreateProps { open: boolean; handleClose: () => void; - onSubmit: (data: { name: string, description: string, files: File[] }) => void; + onSubmit: (data: { name: string, description: string }, files: UploadFile[]) => void; handleSubmit: any; submitting: boolean; invalid: boolean; pristine: boolean; - files: File[]; + files: UploadFile[]; } interface TextFieldProps { @@ -66,13 +66,13 @@ interface TextFieldProps { export const DialogCollectionCreate = compose( connect((state: RootState) => ({ - files: state.collections.uploader.files + files: state.collections.uploader })), reduxForm({ form: 'collectionCreateDialog' }), withStyles(styles))( class DialogCollectionCreate extends React.Component> { render() { - const { classes, open, handleClose, handleSubmit, onSubmit, submitting, invalid, pristine } = this.props; + const { classes, open, handleClose, handleSubmit, onSubmit, submitting, invalid, pristine, files } = this.props; return ( -
onSubmit({ ...data, files: this.props.files }))}> + onSubmit(data, files))}> Create a collection this.props.dispatch(collectionUploaderActions.SET_UPLOAD_FILES(files))}/> -- 2.30.2