src/services/collection-service/collection-service.ts

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 import * as _ from "lodash";
   6 import { CommonResourceService } from "~/common/api/common-resource-service";
   7 import { CollectionResource } from "~/models/collection";
   8 import axios, { AxiosInstance } from "axios";
   9 import { KeepService } from "../keep-service/keep-service";
  10 import { WebDAV } from "~/common/webdav";
  11 import { AuthService } from "../auth-service/auth-service";
  12 import { mapTree, getNodeChildren, getNode, TreeNode } from "../../models/tree";
  13 import { getTagValue } from "~/common/xml";
  14 import { FilterBuilder } from "~/common/api/filter-builder";
  15 import { CollectionFile, createCollectionFile, CollectionFileType, CollectionDirectory, createCollectionDirectory } from '~/models/collection-file';
  16 import { parseKeepManifestText, stringifyKeepManifest } from "../collection-files-service/collection-manifest-parser";
  17 import { KeepManifestStream } from "~/models/keep-manifest";
  18 import { createCollectionFilesTree } from '~/models/collection-file';
  19
  20 export type UploadProgress = (fileId: number, loaded: number, total: number, currentTime: number) => void;
  21
  22 export class CollectionService extends CommonResourceService<CollectionResource> {
  23     constructor(serverApi: AxiosInstance, private keepService: KeepService, private webdavClient: WebDAV, private authService: AuthService) {
  24         super(serverApi, "collections");
  25     }
  26
  27     async files(uuid: string) {
  28         const request = await this.webdavClient.propfind(`/c=${uuid}`);
  29         if (request.responseXML != null) {
  30             const files = this.extractFilesData(request.responseXML);
  31             const tree = createCollectionFilesTree(files);
  32             const sortedTree = mapTree(node => {
  33                 const children = getNodeChildren(node.id)(tree).map(id => getNode(id)(tree)) as TreeNode<CollectionDirectory | CollectionFile>[];
  34                 children.sort((a, b) =>
  35                     a.value.type !== b.value.type
  36                         ? a.value.type === CollectionFileType.DIRECTORY ? -1 : 1
  37                         : a.value.name.localeCompare(b.value.name)
  38                 );
  39                 return { ...node, children: children.map(child => child.id) };
  40             })(tree);
  41             return sortedTree;
  42         }
  43         return Promise.reject();
  44     }
  45
  46     async deleteFile(collectionUuid: string, filePath: string) {
  47         return this.webdavClient.delete(`/c=${collectionUuid}${filePath}`);
  48     }
  49
  50     extractFilesData(document: Document) {
  51         const collectionUrlPrefix = /\/c=[0-9a-zA-Z\-]*/;
  52         return Array
  53             .from(document.getElementsByTagName('D:response'))
  54             .slice(1) // omit first element which is collection itself
  55             .map(element => {
  56                 const name = getTagValue(element, 'D:displayname', '');
  57                 const size = parseInt(getTagValue(element, 'D:getcontentlength', '0'), 10);
  58                 const pathname = getTagValue(element, 'D:href', '');
  59                 const nameSuffix = `/${name || ''}`;
  60                 const directory = pathname
  61                     .replace(collectionUrlPrefix, '')
  62                     .replace(nameSuffix, '');
  63                 const href = this.webdavClient.defaults.baseURL + pathname + '?api_token=' + this.authService.getApiToken();
  64
  65                 const data = {
  66                     url: href,
  67                     id: `${directory}/${name}`,
  68                     name,
  69                     path: directory,
  70                 };
  71
  72                 return getTagValue(element, 'D:resourcetype', '')
  73                     ? createCollectionDirectory(data)
  74                     : createCollectionFile({ ...data, size });
  75
  76             });
  77     }
  78
  79     private readFile(file: File): Promise<ArrayBuffer> {
  80         return new Promise<ArrayBuffer>(resolve => {
  81             const reader = new FileReader();
  82             reader.onload = () => {
  83                 resolve(reader.result as ArrayBuffer);
  84             };
  85
  86             reader.readAsArrayBuffer(file);
  87         });
  88     }
  89
  90     private uploadFile(keepServiceHost: string, file: File, fileId: number, onProgress?: UploadProgress): Promise<CollectionFile> {
  91         return this.readFile(file).then(content => {
  92             return axios.post<string>(keepServiceHost, content, {
  93                 headers: {
  94                     'Content-Type': 'text/octet-stream'
  95                 },
  96                 onUploadProgress: (e: ProgressEvent) => {
  97                     if (onProgress) {
  98                         onProgress(fileId, e.loaded, e.total, Date.now());
  99                     }
 100                     console.log(`${e.loaded} / ${e.total}`);
 101                 }
 102             }).then(data => createCollectionFile({
 103                 id: data.data,
 104                 name: file.name,
 105                 size: file.size
 106             }));
 107         });
 108     }
 109
 110     private async updateManifest(collectionUuid: string, files: CollectionFile[]): Promise<CollectionResource> {
 111         const collection = await this.get(collectionUuid);
 112         const manifest: KeepManifestStream[] = parseKeepManifestText(collection.manifestText);
 113
 114         files.forEach(f => {
 115             let kms = manifest.find(stream => stream.name === f.path);
 116             if (!kms) {
 117                 kms = {
 118                     files: [],
 119                     locators: [],
 120                     name: f.path
 121                 };
 122                 manifest.push(kms);
 123             }
 124             kms.locators.push(f.id);
 125             const len = kms.files.length;
 126             const nextPos = len > 0
 127                 ? parseInt(kms.files[len - 1].position, 10) + kms.files[len - 1].size
 128                 : 0;
 129             kms.files.push({
 130                 name: f.name,
 131                 position: nextPos.toString(),
 132                 size: f.size
 133             });
 134         });
 135
 136         console.log(manifest);
 137
 138         const manifestText = stringifyKeepManifest(manifest);
 139         const data = { ...collection, manifestText };
 140         return this.update(collectionUuid, CommonResourceService.mapKeys(_.snakeCase)(data));
 141     }
 142
 143     uploadFiles(collectionUuid: string, files: File[], onProgress?: UploadProgress): Promise<CollectionResource | never> {
 144         const filters = new FilterBuilder()
 145             .addEqual("service_type", "proxy");
 146
 147         return this.keepService.list({ filters: filters.getFilters() }).then(data => {
 148             if (data.items && data.items.length > 0) {
 149                 const serviceHost =
 150                     (data.items[0].serviceSslFlag ? "https://" : "http://") +
 151                     data.items[0].serviceHost +
 152                     ":" + data.items[0].servicePort;
 153
 154                 console.log("serviceHost", serviceHost);
 155
 156                 const files$ = files.map((f, idx) => this.uploadFile(serviceHost, f, idx, onProgress));
 157                 return Promise.all(files$).then(values => {
 158                     return this.updateManifest(collectionUuid, values);
 159                 });
 160             } else {
 161                 return Promise.reject("Missing keep service host");
 162             }
 163         });
 164     }
 165
 166     trash(uuid: string): Promise<CollectionResource> {
 167         return this.serverApi
 168             .post(this.resourceType + `${uuid}/trash`)
 169             .then(CommonResourceService.mapResponseKeys);
 170     }
 171
 172     untrash(uuid: string): Promise<CollectionResource> {
 173         const params = {
 174             ensure_unique_name: true
 175         };
 176         return this.serverApi
 177             .post(this.resourceType + `${uuid}/untrash`, {
 178                 params: CommonResourceService.mapKeys(_.snakeCase)(params)
 179             })
 180             .then(CommonResourceService.mapResponseKeys);
 181     }
 182
 183 }