apps/workbench/app/assets/javascripts/upload_to_collection.js

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 var app = angular.module('Workbench', ['Arvados']);
   6 app.controller('UploadToCollection', UploadToCollection);
   7 app.directive('arvUuid', arvUuid);
   8
   9 function arvUuid() {
  10     // Copy the given uuid into the current $scope.
  11     return {
  12         restrict: 'A',
  13         link: function(scope, element, attributes) {
  14             scope.uuid = attributes.arvUuid;
  15         }
  16     };
  17 }
  18
  19 UploadToCollection.$inject = ['$scope', '$filter', '$q', '$timeout',
  20                               'ArvadosClient', 'arvadosApiToken'];
  21 function UploadToCollection($scope, $filter, $q, $timeout,
  22                             ArvadosClient, arvadosApiToken) {
  23     $.extend($scope, {
  24         uploadQueue: [],
  25         uploader: new QueueUploader(),
  26         addFilesToQueue: function(files) {
  27             // Angular binding doesn't work its usual magic for file
  28             // inputs, so we need to $scope.$apply() this update.
  29             $scope.$apply(function(){
  30                 var i, nItemsTodo;
  31                 // Add these new files after the items already waiting
  32                 // in the queue -- but before the items that are
  33                 // 'Done' and have therefore been pushed to the
  34                 // bottom.
  35                 for (nItemsTodo = 0;
  36                      (nItemsTodo < $scope.uploadQueue.length &&
  37                       $scope.uploadQueue[nItemsTodo].state !== 'Done'); ) {
  38                     nItemsTodo++;
  39                 }
  40                 for (i=0; i<files.length; i++) {
  41                     $scope.uploadQueue.splice(nItemsTodo+i, 0,
  42                         new FileUploader(files[i]));
  43                 }
  44             });
  45         },
  46         go: function() {
  47             $scope.uploader.go();
  48         },
  49         stop: function() {
  50             $scope.uploader.stop();
  51         },
  52         removeFileFromQueue: function(index) {
  53             var wasRunning = $scope.uploader.running;
  54             $scope.uploadQueue[index].stop();
  55             $scope.uploadQueue.splice(index, 1);
  56             if (wasRunning)
  57                 $scope.go();
  58         },
  59         countInStates: function(want_states) {
  60             var found = 0;
  61             $.each($scope.uploadQueue, function() {
  62                 if (want_states.indexOf(this.state) >= 0) {
  63                     ++found;
  64                 }
  65             });
  66             return found;
  67         }
  68     });
  69     ////////////////////////////////
  70
  71     var keepProxy;
  72     var defaultErrorMessage = 'A network error occurred: either the server was unreachable, or there is a server configuration problem. Please check your browser debug console for a more specific error message (browser security features prevent us from showing the details here).';
  73
  74     function SliceReader(_slice) {
  75         var that = this;
  76         $.extend(this, {
  77             go: go
  78         });
  79         ////////////////////////////////
  80         var _deferred;
  81         var _reader;
  82         function go() {
  83             // Return a promise, which will be resolved with the
  84             // requested slice data.
  85             _deferred = $.Deferred();
  86             _reader = new FileReader();
  87             _reader.onload = resolve;
  88             _reader.onerror = _deferred.reject;
  89             _reader.onprogress = _deferred.notify;
  90             _reader.readAsArrayBuffer(_slice.blob);
  91             return _deferred.promise();
  92         }
  93         function resolve() {
  94             if (that._reader.result.length !== that._slice.size) {
  95                 // Sometimes we get an onload event even if the read
  96                 // did not return the desired number of bytes. We
  97                 // treat that as a fail.
  98                 _deferred.reject(
  99                     null, "Read error",
 100                     "Short read: wanted " + _slice.size +
 101                         ", received " + _reader.result.length);
 102                 return;
 103             }
 104             return _deferred.resolve(_reader.result);
 105         }
 106     }
 107
 108     function SliceUploader(_label, _data, _dataSize) {
 109         $.extend(this, {
 110             go: go,
 111             stop: stop
 112         });
 113         ////////////////////////////////
 114         var that = this;
 115         var _deferred;
 116         var _failCount = 0;
 117         var _failMax = 3;
 118         var _jqxhr;
 119         function go() {
 120             // Send data to the Keep proxy. Retry a few times on
 121             // fail. Return a promise that will get resolved with
 122             // resolve(locator) when the block is accepted by the
 123             // proxy.
 124             _deferred = $.Deferred();
 125             if (proxyUriBase().match(/^http:/) &&
 126                 window.location.origin.match(/^https:/)) {
 127                 // In this case, requests will fail, and no ajax
 128                 // success/fail handlers will be called (!), which
 129                 // will leave our status saying "uploading" and the
 130                 // user waiting for something to happen. Better to
 131                 // give up now.
 132                 _deferred.reject({
 133                     textStatus: 'error',
 134                     err: 'There is a server configuration problem. Proxy ' + proxyUriBase() + ' cannot be used from origin ' + window.location.origin + ' due to the browser\'s mixed-content (https/http) policy.'
 135                 });
 136             } else {
 137                 goSend();
 138             }
 139             return _deferred.promise();
 140         }
 141         function stop() {
 142             _failMax = 0;
 143             _jqxhr.abort();
 144             _deferred.reject({
 145                 textStatus: 'stopped',
 146                 err: 'interrupted at slice '+_label
 147             });
 148         }
 149         function goSend() {
 150             _jqxhr = $.ajax({
 151                 url: proxyUriBase(),
 152                 type: 'POST',
 153                 crossDomain: true,
 154                 headers: {
 155                     'Authorization': 'OAuth2 '+arvadosApiToken,
 156                     'Content-Type': 'application/octet-stream',
 157                     'X-Keep-Desired-Replicas': '2'
 158                 },
 159                 xhr: function() {
 160                     // Make an xhr that reports upload progress
 161                     var xhr = $.ajaxSettings.xhr();
 162                     if (xhr.upload) {
 163                         xhr.upload.onprogress = onSendProgress;
 164                     }
 165                     return xhr;
 166                 },
 167                 processData: false,
 168                 data: _data
 169             });
 170             _jqxhr.then(onSendResolve, onSendReject);
 171         }
 172         function onSendProgress(xhrProgressEvent) {
 173             _deferred.notify(xhrProgressEvent.loaded, _dataSize);
 174         }
 175         function onSendResolve(data, textStatus, jqxhr) {
 176             _deferred.resolve(data, _dataSize);
 177         }
 178         function onSendReject(xhr, textStatus, err) {
 179             if (++_failCount < _failMax) {
 180                 // TODO: nice to tell the user that retry is happening.
 181                 console.log('slice ' + _label + ': ' +
 182                             textStatus + ', retry ' + _failCount);
 183                 goSend();
 184             } else {
 185                 _deferred.reject(
 186                     {xhr: xhr, textStatus: textStatus, err: err});
 187             }
 188         }
 189         function proxyUriBase() {
 190             return ((keepProxy.service_ssl_flag ? 'https' : 'http') +
 191                     '://' + keepProxy.service_host + ':' +
 192                     keepProxy.service_port + '/');
 193         }
 194     }
 195
 196     function FileUploader(file) {
 197         $.extend(this, {
 198             file: file,
 199             locators: [],
 200             progress: 0.0,
 201             state: 'Queued',    // Queued, Uploading, Paused, Uploaded, Done
 202             statistics: null,
 203             go: go,
 204             stop: stop          // User wants to stop.
 205         });
 206         ////////////////////////////////
 207         var that = this;
 208         var _currentUploader;
 209         var _currentSlice;
 210         var _deferred;
 211         var _maxBlobSize = Math.pow(2,26);
 212         var _bytesDone = 0;
 213         var _queueTime = Date.now();
 214         var _startTime;
 215         var _startByte;
 216         var _finishTime;
 217         var _readPos = 0;       // number of bytes confirmed uploaded
 218         function go() {
 219             if (_deferred)
 220                 _deferred.reject({textStatus: 'restarted'});
 221             _deferred = $.Deferred();
 222             that.state = 'Uploading';
 223             _startTime = Date.now();
 224             _startByte = _readPos;
 225             setProgress();
 226             goSlice();
 227             return _deferred.promise().always(function() { _deferred = null; });
 228         }
 229         function stop() {
 230             if (_deferred) {
 231                 that.state = 'Paused';
 232                 _deferred.reject({textStatus: 'stopped', err: 'interrupted'});
 233             }
 234             if (_currentUploader) {
 235                 _currentUploader.stop();
 236                 _currentUploader = null;
 237             }
 238         }
 239         function goSlice() {
 240             // Ensure this._deferred gets resolved or rejected --
 241             // either right here, or when a new promise arranged right
 242             // here is fulfilled.
 243             _currentSlice = nextSlice();
 244             if (!_currentSlice) {
 245                 // All slices have been uploaded, but the work won't
 246                 // be truly Done until the target collection has been
 247                 // updated by the QueueUploader. This state is called:
 248                 that.state = 'Uploaded';
 249                 setProgress(_readPos);
 250                 _currentUploader = null;
 251                 _deferred.resolve([that]);
 252                 return;
 253             }
 254             _currentUploader = new SliceUploader(
 255                 _readPos.toString(),
 256                 _currentSlice.blob,
 257                 _currentSlice.size);
 258             _currentUploader.go().then(
 259                 onUploaderResolve,
 260                 onUploaderReject,
 261                 onUploaderProgress);
 262         }
 263         function onUploaderResolve(locator, dataSize) {
 264             var sizeHint = (''+locator).split('+')[1];
 265             if (!locator || parseInt(sizeHint) !== dataSize) {
 266                 console.log("onUploaderResolve, but locator '" + locator +
 267                             "' with size hint '" + sizeHint +
 268                             "' does not look right for dataSize=" + dataSize);
 269                 return onUploaderReject({
 270                     textStatus: "error",
 271                     err: "Bad response from slice upload"
 272                 });
 273             }
 274             that.locators.push(locator);
 275             _readPos += dataSize;
 276             _currentUploader = null;
 277             goSlice();
 278         }
 279         function onUploaderReject(reason) {
 280             that.state = 'Paused';
 281             setProgress(_readPos);
 282             _currentUploader = null;
 283             if (_deferred)
 284                 _deferred.reject(reason);
 285         }
 286         function onUploaderProgress(sliceDone, sliceSize) {
 287             setProgress(_readPos + sliceDone);
 288         }
 289         function nextSlice() {
 290             var size = Math.min(
 291                 _maxBlobSize,
 292                 that.file.size - _readPos);
 293             setProgress(_readPos);
 294             if (size === 0) {
 295                 return false;
 296             }
 297             var blob = that.file.slice(
 298                 _readPos, _readPos+size,
 299                 'application/octet-stream; charset=x-user-defined');
 300             return {blob: blob, size: size};
 301         }
 302         function setProgress(bytesDone) {
 303             var kBps;
 304             if (that.file.size == 0)
 305                 that.progress = 100;
 306             else
 307                 that.progress = Math.min(100, 100 * bytesDone / that.file.size);
 308             if (bytesDone > _startByte) {
 309                 kBps = (bytesDone - _startByte) /
 310                     (Date.now() - _startTime);
 311                 that.statistics = (
 312                     '' + $filter('number')(bytesDone/1024, '0') + ' KiB ' +
 313                         'at ~' + $filter('number')(kBps, '0') + ' KiB/s')
 314                 if (that.state === 'Paused') {
 315                     that.statistics += ', paused';
 316                 } else if (that.state === 'Uploading') {
 317                     that.statistics += ', ETA ' +
 318                         $filter('date')(
 319                             new Date(
 320                                 Date.now() + (that.file.size - bytesDone) / kBps),
 321                             'shortTime')
 322                 }
 323             } else {
 324                 that.statistics = that.state;
 325             }
 326             if (that.state === 'Uploaded') {
 327                 // 'Uploaded' gets reported as 'finished', which is a
 328                 // little misleading because the collection hasn't
 329                 // been updated yet. But FileUploader's portion of the
 330                 // work (and the time when it makes sense to show
 331                 // speed and ETA) is finished.
 332                 that.statistics += ', finished ' +
 333                     $filter('date')(Date.now(), 'shortTime');
 334                 _finishTime = Date.now();
 335             }
 336             if (_deferred)
 337                 _deferred.notify();
 338         }
 339     }
 340
 341     function QueueUploader() {
 342         $.extend(this, {
 343             state: 'Idle',      // Idle, Running, Stopped, Failed
 344             stateReason: null,
 345             statusSuccess: null,
 346             go: go,
 347             stop: stop
 348         });
 349         ////////////////////////////////
 350         var that = this;
 351         var _deferred;          // the one we promise to go()'s caller
 352         var _deferredAppend;    // tracks current appendToCollection
 353         function go() {
 354             if (_deferred) return _deferred.promise();
 355             if (_deferredAppend) return _deferredAppend.promise();
 356             _deferred = $.Deferred();
 357             that.state = 'Running';
 358             ArvadosClient.apiPromise(
 359                 'keep_services', 'list',
 360                 {filters: [['service_type','=','proxy']]}).
 361                 then(doQueueWithProxy);
 362             onQueueProgress();
 363             return _deferred.promise().always(function() { _deferred = null; });
 364         }
 365         function stop() {
 366             that.state = 'Stopped';
 367             if (_deferred) {
 368                 _deferred.reject({});
 369             }
 370             for (var i=0; i<$scope.uploadQueue.length; i++)
 371                 $scope.uploadQueue[i].stop();
 372             onQueueProgress();
 373         }
 374         function doQueueWithProxy(data) {
 375             keepProxy = data.items[0];
 376             if (!keepProxy) {
 377                 that.state = 'Failed';
 378                 that.stateReason =
 379                     'There seems to be no Keep proxy service available.';
 380                 _deferred.reject(null, 'error', that.stateReason);
 381                 return;
 382             }
 383             return doQueueWork();
 384         }
 385         function doQueueWork() {
 386             // If anything is not Done, do it.
 387             if ($scope.uploadQueue.length > 0 &&
 388                 $scope.uploadQueue[0].state !== 'Done') {
 389                 if (_deferred) {
 390                     that.stateReason = null;
 391                     return $scope.uploadQueue[0].go().
 392                         then(appendToCollection, null, onQueueProgress).
 393                         then(doQueueWork, onQueueReject);
 394                 } else {
 395                     // Queue work has been stopped. Just update the
 396                     // view.
 397                     onQueueProgress();
 398                     return;
 399                 }
 400             }
 401             // If everything is Done, resolve the promise and clean
 402             // up. Note this can happen even after the _deferred
 403             // promise has been rejected: specifically, when stop() is
 404             // called too late to prevent completion of the last
 405             // upload. In that case we want to update state to "Idle",
 406             // rather than leave it at "Stopped".
 407             onQueueResolve();
 408         }
 409         function onQueueReject(reason) {
 410             if (!_deferred) {
 411                 // Outcome has already been decided (by stop()).
 412                 return;
 413             }
 414
 415             that.state = 'Failed';
 416             that.stateReason = (
 417                 (reason.textStatus || 'Error') +
 418                     (reason.xhr && reason.xhr.options
 419                      ? (' (from ' + reason.xhr.options.url + ')')
 420                      : '') +
 421                     ': ' +
 422                     (reason.err || defaultErrorMessage));
 423             if (reason.xhr && reason.xhr.responseText)
 424                 that.stateReason += ' -- ' + reason.xhr.responseText;
 425             _deferred.reject(reason);
 426             onQueueProgress();
 427         }
 428         function onQueueResolve() {
 429             that.state = 'Idle';
 430             that.stateReason = 'Done!';
 431             if (_deferred)
 432                 _deferred.resolve();
 433             onQueueProgress();
 434         }
 435         function onQueueProgress() {
 436             // Ensure updates happen after FileUpload promise callbacks.
 437             $timeout(function(){$scope.$apply();});
 438         }
 439         function appendToCollection(uploads) {
 440             _deferredAppend = $.Deferred();
 441             ArvadosClient.apiPromise(
 442                 'collections', 'get',
 443                 { uuid: $scope.uuid }).
 444                 then(function(collection) {
 445                     var manifestText = '';
 446                     $.each(uploads, function(_, upload) {
 447                         var locators = upload.locators;
 448                         if (locators.length === 0) {
 449                             // Every stream must have at least one
 450                             // data locator, even if it is zero bytes
 451                             // long:
 452                             locators = ['d41d8cd98f00b204e9800998ecf8427e+0'];
 453                         }
 454                         filename = ArvadosClient.uniqueNameForManifest(
 455                             collection.manifest_text,
 456                             '.', upload.file.name);
 457                         collection.manifest_text += '. ' +
 458                             locators.join(' ') +
 459                             ' 0:' + upload.file.size.toString() + ':' +
 460                             filename +
 461                             '\n';
 462                     });
 463                     return ArvadosClient.apiPromise(
 464                         'collections', 'update',
 465                         { uuid: $scope.uuid,
 466                           collection:
 467                           { manifest_text:
 468                             collection.manifest_text }
 469                         });
 470                 }).
 471                 then(function() {
 472                     // Mark the completed upload(s) as Done and push
 473                     // them to the bottom of the queue.
 474                     var i, qLen = $scope.uploadQueue.length;
 475                     for (i=0; i<qLen; i++) {
 476                         if (uploads.indexOf($scope.uploadQueue[i]) >= 0) {
 477                             $scope.uploadQueue[i].state = 'Done';
 478                             $scope.uploadQueue.push.apply(
 479                                 $scope.uploadQueue,
 480                                 $scope.uploadQueue.splice(i, 1));
 481                             --i;
 482                             --qLen;
 483                         }
 484                     }
 485                 }).
 486                 then(_deferredAppend.resolve,
 487                      _deferredAppend.reject);
 488             return _deferredAppend.promise().
 489                 always(function() {
 490                     _deferredAppend = null;
 491                 });
 492         }
 493     }
 494 }