src/services/collection-service/collection-service.ts

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 import { CommonResourceService } from "../../common/api/common-resource-service";
   6 import { CollectionResource } from "../../models/collection";
   7 import axios, { AxiosInstance } from "axios";
   8 import { KeepService } from "../keep-service/keep-service";
   9 import { FilterBuilder } from "../../common/api/filter-builder";
  10 import { CollectionFile, createCollectionFile, createCollectionDirectory, createCollectionFilesTree, CollectionFileType, CollectionDirectory } from "../../models/collection-file";
  11 import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
  12 import * as _ from "lodash";
  13 import { KeepManifestStream } from "../../models/keep-manifest";
  14 import { WebDAV } from "../../common/webdav";
  15 import { AuthService } from "../auth-service/auth-service";
  16 import { mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
  17 import { getTagValue } from "../../common/xml";
  18
  19 export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
  20
  21 export class CollectionService extends CommonResourceService<CollectionResource> {
  22     constructor(serverApi: AxiosInstance, private keepService: KeepService, private webdavClient: WebDAV, private authService: AuthService) {
  23         super(serverApi, "collections");
  24     }
  25
  26     async files(uuid: string) {
  27         const request = await this.webdavClient.propfind(`/c=${uuid}`);
  28         if (request.responseXML != null) {
  29             const files = this.extractFilesData(request.responseXML);
  30             const tree = createCollectionFilesTree(files);
  31             const sortedTree = mapTree(node => {
  32                 const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
  33                 children.sort((a, b) =>
  34                     a.value.type !== b.value.type
  35                         ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
  36                         : a.value.name.localeCompare(b.value.name)
  37                 );
  38                 return { ...node, children: children.map(child => child.id) };
  39             })(tree);
  40             return sortedTree;
  41         }
  42         return Promise.reject();
  43     }
  44
  45     async deleteFile(collectionUuid: string, filePath: string) {
  46         return this.webdavClient.delete(`/c=${collectionUuid}${filePath}`);
  47     }
  48
  49     extractFilesData(document: Document) {
  50         return Array
  51             .from(document.getElementsByTagName('D:response'))
  52             .slice(1)
  53             .map(element => {
  54                 const name = getTagValue(element, 'D:displayname', '');
  55                 const size = parseInt(getTagValue(element, 'D:getcontentlength', '0') as string, 10);
  56                 const pathname = getTagValue(element, 'D:href', '');
  57                 const directory = pathname && pathname.replace(/\/c=[0-9a-zA-Z\-]*/, '').replace(`/${name || ''}`, '');
  58                 const href = this.webdavClient.defaults.baseURL + pathname + '?api_token=' + this.authService.getApiToken();
  59
  60                 const data = {
  61                     url: href,
  62                     id: `${directory}/${name}`,
  63                     name,
  64                     path: directory,
  65                 };
  66
  67                 const [resourceTypeElement] = Array.from(element.getElementsByTagName('D:resourcetype'));
  68                 return resourceTypeElement && resourceTypeElement.innerHTML === ''
  69                     ? createCollectionFile({ ...data, size })
  70                     : createCollectionDirectory(data);
  71
  72             });
  73     }
  74
  75
  76     private readFile(file: File): Promise<ArrayBuffer> {
  77         return new Promise<ArrayBuffer>(resolve => {
  78             const reader = new FileReader();
  79             reader.onload = () => {
  80                 resolve(reader.result as ArrayBuffer);
  81             };
  82
  83             reader.readAsArrayBuffer(file);
  84         });
  85     }
  86
  87     private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
  88         return this.readFile(file).then(content => {
  89             return axios.post<string>(keepServiceHost, content, {
  90                 headers: {
  91                     'Content-Type': 'text/octet-stream'
  92                 },
  93                 onUploadProgress: (e: ProgressEvent) => {
  94                     if (onProgress) {
  95                         onProgress(fileId, e.loaded, e.total, Date.now());
  96                     }
  97                     console.log(`${e.loaded} / ${e.total}`);
  98                 }
  99             }).then(data => createCollectionFile({
 100                 id: data.data,
 101                 name: file.name,
 102                 size: file.size
 103             }));
 104         });
 105     }
 106
 107     private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
 108         const collection = await this.get(collectionUuid);
 109         const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
 110
 111         files.forEach(f => {
 112             let kms = manifest.find(stream => stream.name === f.path);
 113             if (!kms) {
 114                 kms = {
 115                     files: [],
 116                     locators: [],
 117                     name: f.path
 118                 };
 119                 manifest.push(kms);
 120             }
 121             kms.locators.push(f.id);
 122             const len = kms.files.length;
 123             const nextPos = len > 0
 124                 ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
 125                 : 0;
 126             kms.files.push({
 127                 name: f.name,
 128                 position: nextPos.toString(),
 129                 size: f.size
 130             });
 131         });
 132
 133         console.log(manifest);
 134
 135         const manifestText = stringifyKeepManifest(manifest);
 136         const data = { ...collection, manifestText };
 137         return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
 138     }
 139
 140     uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
 141         const filters = FilterBuilder.create()
 142             .addEqual("service_type", "proxy");
 143
 144         return this.keepService.list({ filters }).then(data => {
 145             if (data.items && data.items.length > 0) {
 146                 const serviceHost =
 147                     (data.items[0].serviceSslFlag ? "https://" : "http://") +
 148                     data.items[0].serviceHost +
 149                     ":" + data.items[0].servicePort;
 150
 151                 console.log("serviceHost", serviceHost);
 152
 153                 const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
 154                 return Promise.all(files$).then(values => {
 155                     return this.updateManifest(collectionUuid, values);
 156                 });
 157             } else {
 158                 return Promise.reject("Missing keep service host");
 159             }
 160         });
 161     }
 162 }