Extract response parsing utils from collection service, use webdav to upload files...

Feature #14013

Arvados-DCO-1.1-Signed-off-by: Michal Klobukowski <michal.klobukowski@contractors.roche.com>

diff --git a/src/services/collection-service/collection-service-files-response.ts b/src/services/collection-service/collection-service-files-response.ts
new file mode 100644 (file)
index 0000000..4545096
--- /dev/null
+++ b/src/services/collection-service/collection-service-files-response.ts
@@ -0,0 +1,53 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+import { createCollectionFilesTree, CollectionDirectory, CollectionFile, CollectionFileType, createCollectionDirectory, createCollectionFile } from "../../models/collection-file";
+import { Tree, mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
+import { getTagValue } from "../../common/xml";
+
+export const parseFilesResponse = (document: Document) => {
+    const files = extractFilesData(document);
+    const tree = createCollectionFilesTree(files);
+    return sortFilesTree(tree);
+};
+
+export const sortFilesTree = (tree: Tree<CollectionDirectory | CollectionFile>) => {
+    return mapTree(node => {
+        const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
+        children.sort((a, b) =>
+            a.value.type !== b.value.type
+                ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
+                : a.value.name.localeCompare(b.value.name)
+        );
+        return { ...node, children: children.map(child => child.id) } as TreeNode<CollectionDirectory | CollectionFile>;
+    })(tree);
+};
+
+export const extractFilesData = (document: Document) => {
+    const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
+    return Array
+        .from(document.getElementsByTagName('D:response'))
+        .slice(1) // omit first element which is collection itself
+        .map(element => {
+            const name = getTagValue(element, 'D:displayname', '');
+            const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
+            const pathname = getTagValue(element, 'D:href', '');
+            const nameSuffix = `/${name || ''}`;
+            const directory = pathname
+                .replace(collectionUrlPrefix, '')
+                .replace(nameSuffix, '');
+
+            const data = {
+                url: pathname,
+                id: `${directory}/${name}`,
+                name,
+                path: directory,
+            };
+
+            return getTagValue(element, 'D:resourcetype', '')
+                ? createCollectionDirectory(data)
+                : createCollectionFile({ ...data, size });
+
+        });
+};

diff --git a/src/services/collection-service/collection-service.ts b/src/services/collection-service/collection-service.ts
index 8cf515c97853060b94c894b4764c72e466488f95..3b98f750ec1f24233478fd07e562d0ae7532ba4c 100644 (file)
--- a/src/services/collection-service/collection-service.ts
+++ b/src/services/collection-service/collection-service.ts
@@ -4,40 +4,25 @@
 
 import { CommonResourceService } from "../../common/api/common-resource-service";
 import { CollectionResource } from "../../models/collection";
-import axios, { AxiosInstance } from "axios";
-import { KeepService } from "../keep-service/keep-service";
-import { FilterBuilder } from "../../common/api/filter-builder";
-import { CollectionFile, createCollectionFile, createCollectionDirectory, createCollectionFilesTree, CollectionFileType, CollectionDirectory } from "../../models/collection-file";
-import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
-import * as _ from "lodash";
-import { KeepManifestStream } from "../../models/keep-manifest";
+import { AxiosInstance } from "axios";
+import { CollectionFile, CollectionDirectory } from "../../models/collection-file";
 import { WebDAV } from "../../common/webdav";
 import { AuthService } from "../auth-service/auth-service";
-import { mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
-import { getTagValue } from "../../common/xml";
+import { mapTreeValues } from "../../models/tree";
+import { parseFilesResponse } from "./collection-service-files-response";
 
 export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
 
 export class CollectionService extends CommonResourceService<CollectionResource> {
-    constructor(serverApi: AxiosInstance, private keepService: KeepService, private webdavClient: WebDAV, private authService: AuthService) {
+    constructor(serverApi: AxiosInstance, private webdavClient: WebDAV, private authService: AuthService) {
         super(serverApi, "collections");
     }
 
     async files(uuid: string) {
         const request = await this.webdavClient.propfind(`/c=${uuid}`);
         if (request.responseXML != null) {
-            const files = this.extractFilesData(request.responseXML);
-            const tree = createCollectionFilesTree(files);
-            const sortedTree = mapTree(node => {
-                const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
-                children.sort((a, b) =>
-                    a.value.type !== b.value.type
-                        ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
-                        : a.value.name.localeCompare(b.value.name)
-                );
-                return { ...node, children: children.map(child => child.id) };
-            })(tree);
-            return sortedTree;
+            const filesTree = parseFilesResponse(request.responseXML);
+            return mapTreeValues(this.extendFileURL)(filesTree);
         }
         return Promise.reject();
     }
@@ -46,120 +31,38 @@ export class CollectionService extends CommonResourceService<CollectionResource>
         return this.webdavClient.delete(`/c=${collectionUuid}${filePath}`);
     }
 
-    extractFilesData(document: Document) {
-        const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
-        return Array
-            .from(document.getElementsByTagName('D:response'))
-            .slice(1) // omit first element which is collection itself
-            .map(element => {
-                const name = getTagValue(element, 'D:displayname', '');
-                const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
-                const pathname = getTagValue(element, 'D:href', '');
-                const nameSuffix = `/${name || ''}`;
-                const directory = pathname
-                    .replace(collectionUrlPrefix, '')
-                    .replace(nameSuffix, '');
-                const href = this.webdavClient.defaults.baseURL + pathname + '?api_token=' + this.authService.getApiToken();
-
-                const data = {
-                    url: href,
-                    id: `${directory}/${name}`,
-                    name,
-                    path: directory,
-                };
+    async uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress) {
+        for (let idx = 0; idx < files.length; idx++) {
+            await this.uploadFile(collectionUuid, files[idx], idx, onProgress);
+        }
+    }
 
-                return getTagValue(element, 'D:resourcetype', '')
-                    ? createCollectionDirectory(data)
-                    : createCollectionFile({ ...data, size });
+    private extendFileURL = (file: CollectionDirectory | CollectionFile) => ({
+        ...file,
+        url: this.webdavClient.defaults.baseURL + file.url + '?api_token=' + this.authService.getApiToken()
+    })
 
-            });
+    private uploadFile(collectionUuid: string, file: File, fileId: number, onProgress: UploadProgress = () => { return; }) {
+        return this.readFile(file).then(content =>
+            this.webdavClient.put(`/c=${collectionUuid}/${file.name}`, content, {
+                headers: {
+                    'Content-Type': 'text/octet-stream'
+                },
+                onUploadProgress: (e: ProgressEvent) => {
+                    onProgress(fileId, e.loaded, e.total, Date.now());
+                }
+            })
+        );
     }
 
-
     private readFile(file: File): Promise<ArrayBuffer> {
         return new Promise<ArrayBuffer>(resolve => {
             const reader = new FileReader();
             reader.onload = () => {
                 resolve(reader.result as ArrayBuffer);
             };
-
             reader.readAsArrayBuffer(file);
         });
     }
 
-    private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
-        return this.readFile(file).then(content => {
-            return axios.post<string>(keepServiceHost, content, {
-                headers: {
-                    'Content-Type': 'text/octet-stream'
-                },
-                onUploadProgress: (e: ProgressEvent) => {
-                    if (onProgress) {
-                        onProgress(fileId, e.loaded, e.total, Date.now());
-                    }
-                    console.log(`${e.loaded} / ${e.total}`);
-                }
-            }).then(data => createCollectionFile({
-                id: data.data,
-                name: file.name,
-                size: file.size
-            }));
-        });
-    }
-
-    private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
-        const collection = await this.get(collectionUuid);
-        const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
-
-        files.forEach(f => {
-            let kms = manifest.find(stream => stream.name === f.path);
-            if (!kms) {
-                kms = {
-                    files: [],
-                    locators: [],
-                    name: f.path
-                };
-                manifest.push(kms);
-            }
-            kms.locators.push(f.id);
-            const len = kms.files.length;
-            const nextPos = len > 0
-                ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
-                : 0;
-            kms.files.push({
-                name: f.name,
-                position: nextPos.toString(),
-                size: f.size
-            });
-        });
-
-        console.log(manifest);
-
-        const manifestText = stringifyKeepManifest(manifest);
-        const data = { ...collection, manifestText };
-        return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
-    }
-
-    uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
-        const filters = FilterBuilder.create()
-            .addEqual("service_type", "proxy");
-
-        return this.keepService.list({ filters }).then(data => {
-            if (data.items && data.items.length > 0) {
-                const serviceHost =
-                    (data.items[0].serviceSslFlag ? "https://" : "http://") +
-                    data.items[0].serviceHost +
-                    ":" + data.items[0].servicePort;
-
-                console.log("serviceHost", serviceHost);
-
-                const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
-                return Promise.all(files$).then(values => {
-                    return this.updateManifest(collectionUuid, values);
-                });
-            } else {
-                return Promise.reject("Missing keep service host");
-            }
-        });
-    }
 }

diff --git a/src/services/services.ts b/src/services/services.ts
index eedd1240c7f7ced2495b3966591b311810efa1d7..9b6495fbd986d792a4dff7ada1bfab23a4ef20ff 100644 (file)
--- a/src/services/services.ts
+++ b/src/services/services.ts
@@ -30,7 +30,7 @@ export const createServices = (config: Config) => {
     const projectService = new ProjectService(apiClient);
     const linkService = new LinkService(apiClient);
     const favoriteService = new FavoriteService(linkService, groupsService);
-    const collectionService = new CollectionService(apiClient, keepService, webdavClient, authService);
+    const collectionService = new CollectionService(apiClient, webdavClient, authService);
     const tagService = new TagService(linkService);
     const collectionFilesService = new CollectionFilesService(collectionService);
 

diff --git a/src/store/collections/creator/collection-creator-action.ts b/src/store/collections/creator/collection-creator-action.ts
index d0a66b4cc0f629cf60dc935beee2cf3c42aadfbd..984a3c9c7b1fd4ea5c0fc4a0f0a9425787aaef2e 100644 (file)
--- a/src/store/collections/creator/collection-creator-action.ts
+++ b/src/store/collections/creator/collection-creator-action.ts
@@ -17,9 +17,9 @@ export const collectionCreateActions = unionize({
     CREATE_COLLECTION: ofType<{}>(),
     CREATE_COLLECTION_SUCCESS: ofType<{}>(),
 }, {
-    tag: 'type',
-    value: 'payload'
-});
+        tag: 'type',
+        value: 'payload'
+    });
 
 export const createCollection = (collection: Partial<CollectionResource>, files: File[]) =>
     (dispatch: Dispatch, getState: () => RootState, services: ServiceRepository) => {
@@ -34,11 +34,11 @@ export const createCollection = (collection: Partial<CollectionResource>, files:
                     (fileId, loaded, total, currentTime) => {
                         dispatch(collectionUploaderActions.SET_UPLOAD_PROGRESS({ fileId, loaded, total, currentTime }));
                     })
-                .then(collection => {
-                    dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
-                    dispatch(reset('collectionCreateDialog'));
-                    dispatch(collectionUploaderActions.CLEAR_UPLOAD());
-                });
+                    .then(() => {
+                        dispatch(collectionCreateActions.CREATE_COLLECTION_SUCCESS(collection));
+                        dispatch(reset('collectionCreateDialog'));
+                        dispatch(collectionUploaderActions.CLEAR_UPLOAD());
+                    });
                 return collection;
             });
     };